analytics: Fix FillState.last_modified not being updated.

We were updating FillState with FillState.objects.filter(..).update(..),
which does not update the last_modified field (which has auto_now=True).
The correct incantation is the save() method of the actual FillState
object.

analytics/lib/counts.py

@@ -4,7 +4,7 @@ from django.conf import settings
 from datetime import timedelta, datetime
 
 from analytics.models import InstallationCount, RealmCount, \
-    UserCount, StreamCount, BaseCount, FillState, get_fill_state, installation_epoch
+    UserCount, StreamCount, BaseCount, FillState, installation_epoch
 from zerver.models import Realm, UserProfile, Message, Stream, models
 from zerver.lib.timestamp import floor_to_day
 
@@ -58,35 +58,39 @@ class ZerverCountQuery(object):
         self.analytics_table = analytics_table
         self.query = query
 
+def do_update_fill_state(fill_state, end_time, state):
+    # type: (FillState, datetime, int) -> None
+    fill_state.end_time = end_time
+    fill_state.state = state
+    fill_state.save()
+
 def process_count_stat(stat, fill_to_time):
     # type: (CountStat, datetime) -> None
-    fill_state = get_fill_state(stat.property)
+    fill_state = FillState.objects.filter(property=stat.property).first()
     if fill_state is None:
         currently_filled = installation_epoch()
-        FillState.objects.create(property = stat.property,
-                                 end_time = currently_filled,
-                                 state = FillState.DONE)
+        fill_state = FillState.objects.create(property=stat.property,
+                                              end_time=currently_filled,
+                                              state=FillState.DONE)
         logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
-    elif fill_state['state'] == FillState.STARTED:
-        logger.info("UNDO START %s %s" % (stat.property, fill_state['end_time']))
-        do_delete_count_stat_at_hour(stat, fill_state['end_time'])
-        currently_filled = fill_state['end_time'] - timedelta(hours = 1)
-        FillState.objects.filter(property = stat.property). \
-            update(end_time = currently_filled, state = FillState.DONE)
+    elif fill_state.state == FillState.STARTED:
+        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
+        do_delete_count_stat_at_hour(stat, fill_state.end_time)
+        currently_filled = fill_state.end_time - timedelta(hours = 1)
+        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         logger.info("UNDO DONE %s" % (stat.property,))
-    elif fill_state['state'] == FillState.DONE:
-        currently_filled = fill_state['end_time']
+    elif fill_state.state == FillState.DONE:
+        currently_filled = fill_state.end_time
     else:
-        raise ValueError("Unknown value for FillState.state: %s." % fill_state['state'])
+        raise ValueError("Unknown value for FillState.state: %s." % (fill_state.state,))
 
     currently_filled = currently_filled + timedelta(hours = 1)
     while currently_filled <= fill_to_time:
         logger.info("START %s %s %s" % (stat.property, stat.interval, currently_filled))
         start = time.time()
-        FillState.objects.filter(property = stat.property).update(end_time = currently_filled,
-                                                                  state = FillState.STARTED)
+        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
         do_fill_count_stat_at_hour(stat, currently_filled)
-        FillState.objects.filter(property = stat.property).update(state = FillState.DONE)
+        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
         currently_filled = currently_filled + timedelta(hours = 1)
         logger.info("DONE %s %s (%dms)" % (stat.property, stat.interval, (end-start)*1000))

analytics/management/commands/update_analytics_counts.py

@@ -33,6 +33,9 @@ class Command(BaseCommand):
         parser.add_argument('--stat', '-s',
                             type=str,
                             help="CountStat to process. If omitted, all stats are processed.")
+        parser.add_argument('--quiet', '-q',
+                            type=str,
+                            help="Suppress output to stdout.")
 
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None

analytics/models.py

@@ -24,13 +24,6 @@ class FillState(ModelReprMixin, models.Model):
         # type: () -> Text
         return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 
-def get_fill_state(property):
-    # type: (Text) -> Optional[Dict[str, Any]]
-    try:
-        return FillState.objects.filter(property = property).values('end_time', 'state')[0]
-    except IndexError:
-        return None
-
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
 def installation_epoch():

analytics/tests/test_counts.py

@@ -7,7 +7,7 @@ from analytics.lib.counts import CountStat, COUNT_STATS, process_count_stat, \
     zerver_count_message_by_stream, zerver_count_stream_by_realm, \
     do_fill_count_stat_at_hour, ZerverCountQuery
 from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, FillState, get_fill_state, installation_epoch
+    UserCount, StreamCount, FillState, installation_epoch
 
 from zerver.models import Realm, UserProfile, Message, Stream, Recipient, \
     get_user_profile_by_email, get_client
@@ -94,9 +94,9 @@ class TestProcessCountStat(AnalyticsTestCase):
         count_stat = self.make_dummy_count_stat(end_time)
         if property is None:
             property = count_stat.property
-        fill_state = get_fill_state(property)
-        self.assertEqual(fill_state['end_time'], end_time)
-        self.assertEqual(fill_state['state'], state)
+        fill_state = FillState.objects.filter(property=property).first()
+        self.assertEqual(fill_state.end_time, end_time)
+        self.assertEqual(fill_state.state, state)
 
     def test_process_stat(self):
         # type: () -> None