paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-04 05:53:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

counts.py: Rename DataCollector-level operations to be more generic.

Browse Source 
We're about to use these for DependentCountStats that will run SQL queries
on the analytics tables instead of the zerver tables.
This commit is contained in:
Rishi Gupta
2017-04-05 18:30:36 -07:00
committed by Tim Abbott
parent 47cf1d15ba
commit 1e8d2b984d
2 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
analytics/lib/counts.py
View File

@@ -218,7 +218,7 @@ def do_drop_all_analytics_tables():
## DataCollector-level operations ## ## DataCollector-level operations ##
def do_pull_from_zerver(property, start_time, end_time, query, group_by): def do_pull_by_sql_query(property, start_time, end_time, query, group_by):
# type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int # type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int
if group_by is None: if group_by is None:
subgroup = 'NULL' subgroup = 'NULL'
@@ -239,11 +239,11 @@ def do_pull_from_zerver(property, start_time, end_time, query, group_by):
cursor.close() cursor.close()
return rowcount return rowcount
def zerver_data_collector(output_table, query, group_by): def sql_data_collector(output_table, query, group_by):
# type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector # type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector
def pull_function(property, start_time, end_time): def pull_function(property, start_time, end_time):
# type: (str, datetime, datetime) -> int # type: (str, datetime, datetime) -> int
return do_pull_from_zerver(property, start_time, end_time, query, group_by) return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
return DataCollector(output_table, pull_function) return DataCollector(output_table, pull_function)
def do_pull_minutes_active(property, start_time, end_time): def do_pull_minutes_active(property, start_time, end_time):
@@ -428,22 +428,22 @@ count_stream_by_realm_query = """
count_stats_ = [ count_stats_ = [
CountStat('messages_sent:is_bot:hour', CountStat('messages_sent:is_bot:hour',
zerver_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')), sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
CountStat.HOUR), CountStat.HOUR),
CountStat('messages_sent:message_type:day', CountStat('messages_sent:message_type:day',
zerver_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY), sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
CountStat('messages_sent:client:day', CountStat('messages_sent:client:day',
zerver_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')), sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
CountStat.DAY), CountStat.DAY),
CountStat('messages_in_stream:is_bot:day', CountStat('messages_in_stream:is_bot:day',
zerver_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')), sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
CountStat.DAY), CountStat.DAY),
# Sanity check on the bottom two stats. Is only an approximation, # Sanity check on the bottom two stats. Is only an approximation,
# e.g. if a user is deactivated between the end of the day and when this # e.g. if a user is deactivated between the end of the day and when this
# stat is run, they won't be counted. # stat is run, they won't be counted.
CountStat('active_users:is_bot:day', CountStat('active_users:is_bot:day',
zerver_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')), sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
CountStat.DAY, interval=TIMEDELTA_MAX), CountStat.DAY, interval=TIMEDELTA_MAX),
# In RealmCount, 'active_humans_audit::day' should be the partial sum sequence # In RealmCount, 'active_humans_audit::day' should be the partial sum sequence
# of 'active_users_log:is_bot:day', for any realm that started after the # of 'active_users_log:is_bot:day', for any realm that started after the
@@ -451,14 +451,14 @@ count_stats_ = [
# 'active_users_audit:is_bot:day' is the canonical record of which users were # 'active_users_audit:is_bot:day' is the canonical record of which users were
# active on which days (in the UserProfile.is_active sense). # active on which days (in the UserProfile.is_active sense).
CountStat('active_users_audit:is_bot:day', CountStat('active_users_audit:is_bot:day',
zerver_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')), sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
CountStat.DAY), CountStat.DAY),
LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY), LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
# The minutes=15 part is due to the 15 minutes added in # The minutes=15 part is due to the 15 minutes added in
# zerver.lib.actions.do_update_user_activity_interval. # zerver.lib.actions.do_update_user_activity_interval.
CountStat('15day_actives::day', CountStat('15day_actives::day',
zerver_data_collector(UserCount, check_useractivityinterval_by_user_query, None), sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
CountStat.DAY, interval=timedelta(days=15)-timedelta(minutes=15)), CountStat.DAY, interval=timedelta(days=15)-timedelta(minutes=15)),
CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY) CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY)
] ]

4
analytics/tests/test_counts.py
View File

@@ -8,7 +8,7 @@ from django.utils import timezone
from analytics.lib.counts import CountStat, COUNT_STATS, process_count_stat, \ from analytics.lib.counts import CountStat, COUNT_STATS, process_count_stat, \
do_fill_count_stat_at_hour, do_increment_logging_stat, DataCollector, \ do_fill_count_stat_at_hour, do_increment_logging_stat, DataCollector, \
zerver_data_collector, LoggingCountStat, do_aggregate_to_summary_table, \ sql_data_collector, LoggingCountStat, do_aggregate_to_summary_table, \
do_drop_all_analytics_tables do_drop_all_analytics_tables
from analytics.models import BaseCount, InstallationCount, RealmCount, \ from analytics.models import BaseCount, InstallationCount, RealmCount, \
UserCount, StreamCount, FillState, Anomaly, installation_epoch UserCount, StreamCount, FillState, Anomaly, installation_epoch
@@ -157,7 +157,7 @@ class TestProcessCountStat(AnalyticsTestCase):
# type: (datetime) -> CountStat # type: (datetime) -> CountStat
dummy_query = """INSERT INTO analytics_realmcount (realm_id, property, end_time, value) dummy_query = """INSERT INTO analytics_realmcount (realm_id, property, end_time, value)
VALUES (1, 'test stat', '%(end_time)s', 22)""" % {'end_time': current_time} VALUES (1, 'test stat', '%(end_time)s', 22)""" % {'end_time': current_time}
stat = CountStat('test stat', zerver_data_collector(UserCount, dummy_query, None), CountStat.HOUR) stat = CountStat('test stat', sql_data_collector(UserCount, dummy_query, None), CountStat.HOUR)
return stat return stat
def assertFillStateEquals(self, end_time, state=FillState.DONE, property=None): def assertFillStateEquals(self, end_time, state=FillState.DONE, property=None):