paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-03 05:23:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

python: Normalize quotes with Black.

Browse Source 
Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg
2021-02-11 23:20:45 -08:00
committed by Tim Abbott
parent 11741543da
commit 6e4c3e41dc
989 changed files with 32792 additions and 32792 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
analytics/lib/counts.py
View File

@@ -32,7 +32,7 @@ from zerver.models import (
## Logging setup ##
logger = logging.getLogger('zulip.management')
logger = logging.getLogger("zulip.management")
log_to_file(logger, settings.ANALYTICS_LOG_PATH)
# You can't subtract timedelta.max from a datetime, so use this instead
@@ -42,8 +42,8 @@ TIMEDELTA_MAX = timedelta(days=365 * 1000)
class CountStat:
HOUR = 'hour'
DAY = 'day'
HOUR = "hour"
DAY = "day"
FREQUENCIES = frozenset([HOUR, DAY])
@property
@@ -55,7 +55,7 @@ class CountStat:
def __init__(
self,
property: str,
data_collector: 'DataCollector',
data_collector: "DataCollector",
frequency: str,
interval: Optional[timedelta] = None,
) -> None:
@@ -91,7 +91,7 @@ class DependentCountStat(CountStat):
def __init__(
self,
property: str,
data_collector: 'DataCollector',
data_collector: "DataCollector",
frequency: str,
interval: Optional[timedelta] = None,
dependencies: Sequence[str] = [],
@@ -244,8 +244,8 @@ def do_aggregate_to_summary_table(
cursor.execute(
realmcount_query,
{
'property': stat.property,
'end_time': end_time,
"property": stat.property,
"end_time": end_time,
},
)
end = time.time()
@@ -279,8 +279,8 @@ def do_aggregate_to_summary_table(
cursor.execute(
installationcount_query,
{
'property': stat.property,
'end_time': end_time,
"property": stat.property,
"end_time": end_time,
},
)
end = time.time()
@@ -309,11 +309,11 @@ def do_increment_logging_stat(
table = stat.data_collector.output_table
if table == RealmCount:
id_args = {'realm': zerver_object}
id_args = {"realm": zerver_object}
elif table == UserCount:
id_args = {'realm': zerver_object.realm, 'user': zerver_object}
id_args = {"realm": zerver_object.realm, "user": zerver_object}
else: # StreamCount
id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
id_args = {"realm": zerver_object.realm, "stream": zerver_object}
if stat.frequency == CountStat.DAY:
end_time = ceiling_to_day(event_time)
@@ -324,12 +324,12 @@ def do_increment_logging_stat(
property=stat.property,
subgroup=subgroup,
end_time=end_time,
defaults={'value': increment},
defaults={"value": increment},
**id_args,
)
if not created:
row.value = F('value') + increment
row.save(update_fields=['value'])
row.value = F("value") + increment
row.save(update_fields=["value"])
def do_drop_all_analytics_tables() -> None:
@@ -361,11 +361,11 @@ def do_pull_by_sql_query(
group_by: Optional[Tuple[models.Model, str]],
) -> int:
if group_by is None:
subgroup = SQL('NULL')
group_by_clause = SQL('')
subgroup = SQL("NULL")
group_by_clause = SQL("")
else:
subgroup = Identifier(group_by[0]._meta.db_table, group_by[1])
group_by_clause = SQL(', {}').format(subgroup)
group_by_clause = SQL(", {}").format(subgroup)
# We do string replacement here because cursor.execute will reject a
# group_by_clause given as a param.
@@ -373,17 +373,17 @@ def do_pull_by_sql_query(
# think about how to convert python datetimes to SQL datetimes.
query_ = query(
{
'subgroup': subgroup,
'group_by_clause': group_by_clause,
"subgroup": subgroup,
"group_by_clause": group_by_clause,
}
)
cursor = connection.cursor()
cursor.execute(
query_,
{
'property': property,
'time_start': start_time,
'time_end': end_time,
"property": property,
"time_start": start_time,
"time_end": end_time,
},
)
rowcount = cursor.rowcount
@@ -419,9 +419,9 @@ def do_pull_minutes_active(
start__lt=end_time,
)
.select_related(
'user_profile',
"user_profile",
)
.values_list('user_profile_id', 'user_profile__realm_id', 'start', 'end')
.values_list("user_profile_id", "user_profile__realm_id", "start", "end")
)
seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
@@ -713,28 +713,28 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
# Stats that count the number of messages sent in various ways.
# These are also the set of stats that read from the Message table.
CountStat(
'messages_sent:is_bot:hour',
"messages_sent:is_bot:hour",
sql_data_collector(
UserCount, count_message_by_user_query(realm), (UserProfile, 'is_bot')
UserCount, count_message_by_user_query(realm), (UserProfile, "is_bot")
),
CountStat.HOUR,
),
CountStat(
'messages_sent:message_type:day',
"messages_sent:message_type:day",
sql_data_collector(UserCount, count_message_type_by_user_query(realm), None),
CountStat.DAY,
),
CountStat(
'messages_sent:client:day',
"messages_sent:client:day",
sql_data_collector(
UserCount, count_message_by_user_query(realm), (Message, 'sending_client_id')
UserCount, count_message_by_user_query(realm), (Message, "sending_client_id")
),
CountStat.DAY,
),
CountStat(
'messages_in_stream:is_bot:day',
"messages_in_stream:is_bot:day",
sql_data_collector(
StreamCount, count_message_by_stream_query(realm), (UserProfile, 'is_bot')
StreamCount, count_message_by_stream_query(realm), (UserProfile, "is_bot")
),
CountStat.DAY,
),
@@ -744,9 +744,9 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
# active on which days (in the UserProfile.is_active sense).
# Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
CountStat(
'active_users_audit:is_bot:day',
"active_users_audit:is_bot:day",
sql_data_collector(
UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, 'is_bot')
UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
),
CountStat.DAY,
),
@@ -759,15 +759,15 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
# In RealmCount, 'active_users_audit:is_bot:day' should be the partial
# sum sequence of 'active_users_log:is_bot:day', for any realm that
# started after the latter stat was introduced.
LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
# Another sanity check on 'active_users_audit:is_bot:day'. Is only an
# approximation, e.g. if a user is deactivated between the end of the
# day and when this stat is run, they won't be counted. However, is the
# simplest of the three to inspect by hand.
CountStat(
'active_users:is_bot:day',
"active_users:is_bot:day",
sql_data_collector(
RealmCount, count_user_by_realm_query(realm), (UserProfile, 'is_bot')
RealmCount, count_user_by_realm_query(realm), (UserProfile, "is_bot")
),
CountStat.DAY,
interval=TIMEDELTA_MAX,
@@ -779,42 +779,42 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
# as read (imperfect because of batching of some request
# types, but less likely to be overwhelmed by a single bulk
# operation).
LoggingCountStat('messages_read::hour', UserCount, CountStat.HOUR),
LoggingCountStat('messages_read_interactions::hour', UserCount, CountStat.HOUR),
LoggingCountStat("messages_read::hour", UserCount, CountStat.HOUR),
LoggingCountStat("messages_read_interactions::hour", UserCount, CountStat.HOUR),
# User activity stats
# Stats that measure user activity in the UserActivityInterval sense.
CountStat(
'1day_actives::day',
"1day_actives::day",
sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
CountStat.DAY,
interval=timedelta(days=1) - UserActivityInterval.MIN_INTERVAL_LENGTH,
),
CountStat(
'7day_actives::day',
"7day_actives::day",
sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
CountStat.DAY,
interval=timedelta(days=7) - UserActivityInterval.MIN_INTERVAL_LENGTH,
),
CountStat(
'15day_actives::day',
"15day_actives::day",
sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
CountStat.DAY,
interval=timedelta(days=15) - UserActivityInterval.MIN_INTERVAL_LENGTH,
),
CountStat(
'minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
"minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
),
# Rate limiting stats
# Used to limit the number of invitation emails sent by a realm
LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
# Dependent stats
# Must come after their dependencies.
# Canonical account of the number of active humans in a realm on each day.
DependentCountStat(
'realm_active_humans::day',
"realm_active_humans::day",
sql_data_collector(RealmCount, count_realm_active_humans_query(realm), None),
CountStat.DAY,
dependencies=['active_users_audit:is_bot:day', '15day_actives::day'],
dependencies=["active_users_audit:is_bot:day", "15day_actives::day"],
),
]

28
analytics/management/commands/check_analytics_state.py
View File

@@ -26,8 +26,8 @@ class Command(BaseCommand):
def handle(self, *args: Any, **options: Any) -> None:
fill_state = self.get_fill_state()
status = fill_state['status']
message = fill_state['message']
status = fill_state["status"]
message = fill_state["message"]
state_file_path = "/var/lib/nagios_state/check-analytics-state"
state_file_tmp = state_file_path + "-tmp"
@@ -38,7 +38,7 @@ class Command(BaseCommand):
def get_fill_state(self) -> Dict[str, Any]:
if not Realm.objects.exists():
return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
return {"status": 0, "message": "No realms exist, so not checking FillState."}
warning_unfilled_properties = []
critical_unfilled_properties = []
@@ -49,7 +49,7 @@ class Command(BaseCommand):
try:
verify_UTC(last_fill)
except TimezoneNotUTCException:
return {'status': 2, 'message': f'FillState not in UTC for {property}'}
return {"status": 2, "message": f"FillState not in UTC for {property}"}
if stat.frequency == CountStat.DAY:
floor_function = floor_to_day
@@ -62,8 +62,8 @@ class Command(BaseCommand):
if floor_function(last_fill) != last_fill:
return {
'status': 2,
'message': f'FillState not on {stat.frequency} boundary for {property}',
"status": 2,
"message": f"FillState not on {stat.frequency} boundary for {property}",
}
time_to_last_fill = timezone_now() - last_fill
@@ -73,18 +73,18 @@ class Command(BaseCommand):
warning_unfilled_properties.append(property)
if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
return {'status': 0, 'message': 'FillState looks fine.'}
return {"status": 0, "message": "FillState looks fine."}
if len(critical_unfilled_properties) == 0:
return {
'status': 1,
'message': 'Missed filling {} once.'.format(
', '.join(warning_unfilled_properties),
"status": 1,
"message": "Missed filling {} once.".format(
", ".join(warning_unfilled_properties),
),
}
return {
'status': 2,
'message': 'Missed filling {} once. Missed filling {} at least twice.'.format(
', '.join(warning_unfilled_properties),
', '.join(critical_unfilled_properties),
"status": 2,
"message": "Missed filling {} once. Missed filling {} at least twice.".format(
", ".join(warning_unfilled_properties),
", ".join(critical_unfilled_properties),
),
}

4
analytics/management/commands/clear_analytics_tables.py
View File

@@ -10,10 +10,10 @@ class Command(BaseCommand):
help = """Clear analytics tables."""
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--force', action='store_true', help="Clear analytics tables.")
parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
def handle(self, *args: Any, **options: Any) -> None:
if options['force']:
if options["force"]:
do_drop_all_analytics_tables()
else:
raise CommandError(

8
analytics/management/commands/clear_single_stat.py
View File

@@ -10,14 +10,14 @@ class Command(BaseCommand):
help = """Clear analytics tables."""
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--force', action='store_true', help="Actually do it.")
parser.add_argument('--property', help="The property of the stat to be cleared.")
parser.add_argument("--force", action="store_true", help="Actually do it.")
parser.add_argument("--property", help="The property of the stat to be cleared.")
def handle(self, *args: Any, **options: Any) -> None:
property = options['property']
property = options["property"]
if property not in COUNT_STATS:
raise CommandError(f"Invalid property: {property}")
if not options['force']:
if not options["force"]:
raise CommandError("No action taken. Use --force.")
do_drop_single_stat(property)

102
analytics/management/commands/populate_analytics_db.py
View File

@@ -59,7 +59,7 @@ class Command(BaseCommand):
do_drop_all_analytics_tables()
# This also deletes any objects with this realm as a foreign key
Realm.objects.filter(string_id='analytics').delete()
Realm.objects.filter(string_id="analytics").delete()
# Because we just deleted a bunch of objects in the database
# directly (rather than deleting individual objects in Django,
@@ -74,18 +74,18 @@ class Command(BaseCommand):
installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
last_end_time = floor_to_day(timezone_now())
realm = Realm.objects.create(
string_id='analytics', name='Analytics', date_created=installation_time
string_id="analytics", name="Analytics", date_created=installation_time
)
with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time):
shylock = create_user(
'shylock@analytics.ds',
'Shylock',
"shylock@analytics.ds",
"Shylock",
realm,
full_name='Shylock',
full_name="Shylock",
role=UserProfile.ROLE_REALM_ADMINISTRATOR,
)
do_change_user_role(shylock, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None)
stream = Stream.objects.create(name='all', realm=realm, date_created=installation_time)
stream = Stream.objects.create(name="all", realm=realm, date_created=installation_time)
recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
stream.recipient = recipient
stream.save(update_fields=["recipient"])
@@ -108,11 +108,11 @@ class Command(BaseCommand):
if table == InstallationCount:
id_args: Dict[str, Any] = {}
if table == RealmCount:
id_args = {'realm': realm}
id_args = {"realm": realm}
if table == UserCount:
id_args = {'realm': realm, 'user': shylock}
id_args = {"realm": realm, "user": shylock}
if table == StreamCount:
id_args = {'stream': stream, 'realm': realm}
id_args = {"stream": stream, "realm": realm}
for subgroup, values in fixture_data.items():
table.objects.bulk_create(
@@ -127,7 +127,7 @@ class Command(BaseCommand):
if value != 0
)
stat = COUNT_STATS['1day_actives::day']
stat = COUNT_STATS["1day_actives::day"]
realm_data: Mapping[Optional[str], List[int]] = {
None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
}
@@ -140,7 +140,7 @@ class Command(BaseCommand):
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['7day_actives::day']
stat = COUNT_STATS["7day_actives::day"]
realm_data = {
None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
}
@@ -153,7 +153,7 @@ class Command(BaseCommand):
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['realm_active_humans::day']
stat = COUNT_STATS["realm_active_humans::day"]
realm_data = {
None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
}
@@ -166,76 +166,76 @@ class Command(BaseCommand):
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['active_users_audit:is_bot:day']
stat = COUNT_STATS["active_users_audit:is_bot:day"]
realm_data = {
'false': self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
'true': self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
"false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
"true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
}
insert_fixture_data(stat, realm_data, RealmCount)
installation_data = {
'false': self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
'true': self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
"false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
"true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
}
insert_fixture_data(stat, installation_data, InstallationCount)
FillState.objects.create(
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['messages_sent:is_bot:hour']
stat = COUNT_STATS["messages_sent:is_bot:hour"]
user_data: Mapping[Optional[str], List[int]] = {
'false': self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
"false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
}
insert_fixture_data(stat, user_data, UserCount)
realm_data = {
'false': self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
'true': self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
"false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
"true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
}
insert_fixture_data(stat, realm_data, RealmCount)
installation_data = {
'false': self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
'true': self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
"false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
"true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
}
insert_fixture_data(stat, installation_data, InstallationCount)
FillState.objects.create(
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['messages_sent:message_type:day']
stat = COUNT_STATS["messages_sent:message_type:day"]
user_data = {
'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
'private_message': self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
'huddle_message': self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
"public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
"private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
"huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
}
insert_fixture_data(stat, user_data, UserCount)
realm_data = {
'public_stream': self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
'private_stream': self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
'private_message': self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
"public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
"private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
"private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
"huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
}
insert_fixture_data(stat, realm_data, RealmCount)
installation_data = {
'public_stream': self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
'private_stream': self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
'private_message': self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
"public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
"private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
"private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
"huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
}
insert_fixture_data(stat, installation_data, InstallationCount)
FillState.objects.create(
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
website, created = Client.objects.get_or_create(name='website')
old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
android, created = Client.objects.get_or_create(name='ZulipAndroid')
iOS, created = Client.objects.get_or_create(name='ZulipiOS')
react_native, created = Client.objects.get_or_create(name='ZulipMobile')
API, created = Client.objects.get_or_create(name='API: Python')
zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
unused, created = Client.objects.get_or_create(name='unused')
long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
website, created = Client.objects.get_or_create(name="website")
old_desktop, created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
android, created = Client.objects.get_or_create(name="ZulipAndroid")
iOS, created = Client.objects.get_or_create(name="ZulipiOS")
react_native, created = Client.objects.get_or_create(name="ZulipMobile")
API, created = Client.objects.get_or_create(name="API: Python")
zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
unused, created = Client.objects.get_or_create(name="unused")
long_webhook, created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
stat = COUNT_STATS['messages_sent:client:day']
stat = COUNT_STATS["messages_sent:client:day"]
user_data = {
website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
zephyr_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
@@ -269,22 +269,22 @@ class Command(BaseCommand):
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['messages_in_stream:is_bot:day']
stat = COUNT_STATS["messages_in_stream:is_bot:day"]
realm_data = {
'false': self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
'true': self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
"false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
"true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
}
insert_fixture_data(stat, realm_data, RealmCount)
stream_data: Mapping[Optional[str], List[int]] = {
'false': self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
'true': self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
"false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
"true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
}
insert_fixture_data(stat, stream_data, StreamCount)
FillState.objects.create(
property=stat.property, end_time=last_end_time, state=FillState.DONE
)
stat = COUNT_STATS['messages_read::hour']
stat = COUNT_STATS["messages_read::hour"]
user_data = {
None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
}

20
analytics/management/commands/stream_stats.py
View File

@@ -12,13 +12,13 @@ class Command(BaseCommand):
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument(
'realms', metavar='<realm>', nargs='*', help="realm to generate statistics for"
"realms", metavar="<realm>", nargs="*", help="realm to generate statistics for"
)
def handle(self, *args: Any, **options: str) -> None:
if options['realms']:
if options["realms"]:
try:
realms = [get_realm(string_id) for string_id in options['realms']]
realms = [get_realm(string_id) for string_id in options["realms"]]
except Realm.DoesNotExist as e:
raise CommandError(e)
else:
@@ -36,26 +36,26 @@ class Command(BaseCommand):
else:
public_count += 1
print("------------")
print(realm.string_id, end=' ')
print("{:>10} {} public streams and".format("(", public_count), end=' ')
print(realm.string_id, end=" ")
print("{:>10} {} public streams and".format("(", public_count), end=" ")
print(f"{private_count} private streams )")
print("------------")
print("{:>25} {:>15} {:>10} {:>12}".format("stream", "subscribers", "messages", "type"))
for stream in streams:
if stream.invite_only:
stream_type = 'private'
stream_type = "private"
else:
stream_type = 'public'
print(f"{stream.name:>25}", end=' ')
stream_type = "public"
print(f"{stream.name:>25}", end=" ")
recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
print(
"{:10}".format(
len(Subscription.objects.filter(recipient=recipient, active=True))
),
end=' ',
end=" ",
)
num_messages = len(Message.objects.filter(recipient=recipient))
print(f"{num_messages:12}", end=' ')
print(f"{num_messages:12}", end=" ")
print(f"{stream_type:>15}")
print("")

28
analytics/management/commands/update_analytics_counts.py
View File

@@ -23,18 +23,18 @@ class Command(BaseCommand):
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument(
'--time',
'-t',
help='Update stat tables from current state to '
'--time. Defaults to the current time.',
"--time",
"-t",
help="Update stat tables from current state to "
"--time. Defaults to the current time.",
default=timezone_now().isoformat(),
)
parser.add_argument('--utc', action='store_true', help="Interpret --time in UTC.")
parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
parser.add_argument(
'--stat', '-s', help="CountStat to process. If omitted, all stats are processed."
"--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
)
parser.add_argument(
'--verbose', action='store_true', help="Print timing information to stdout."
"--verbose", action="store_true", help="Print timing information to stdout."
)
def handle(self, *args: Any, **options: Any) -> None:
@@ -59,8 +59,8 @@ class Command(BaseCommand):
logger.info("No realms, stopping update_analytics_counts")
return
fill_to_time = parse_datetime(options['time'])
if options['utc']:
fill_to_time = parse_datetime(options["time"])
if options["utc"]:
fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
if fill_to_time.tzinfo is None:
raise ValueError(
@@ -69,23 +69,23 @@ class Command(BaseCommand):
fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
if options['stat'] is not None:
stats = [COUNT_STATS[options['stat']]]
if options["stat"] is not None:
stats = [COUNT_STATS[options["stat"]]]
else:
stats = list(COUNT_STATS.values())
logger.info("Starting updating analytics counts through %s", fill_to_time)
if options['verbose']:
if options["verbose"]:
start = time.time()
last = start
for stat in stats:
process_count_stat(stat, fill_to_time)
if options['verbose']:
if options["verbose"]:
print(f"Updated {stat.property} in {time.time() - last:.3f}s")
last = time.time()
if options['verbose']:
if options["verbose"]:
print(
f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
)

144
analytics/migrations/0001_initial.py
View File

@@ -6,54 +6,54 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('zerver', '0030_realm_org_type'),
("zerver", "0030_realm_org_type"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Anomaly',
name="Anomaly",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
('info', models.CharField(max_length=1000)),
("info", models.CharField(max_length=1000)),
],
bases=(models.Model,),
),
migrations.CreateModel(
name='HuddleCount',
name="HuddleCount",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
(
'huddle',
"huddle",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient'
on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
),
),
(
'user',
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
),
),
('property', models.CharField(max_length=40)),
('end_time', models.DateTimeField()),
('interval', models.CharField(max_length=20)),
('value', models.BigIntegerField()),
("property", models.CharField(max_length=40)),
("end_time", models.DateTimeField()),
("interval", models.CharField(max_length=20)),
("value", models.BigIntegerField()),
(
'anomaly',
"anomaly",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to='analytics.Anomaly',
to="analytics.Anomaly",
null=True,
),
),
@@ -61,23 +61,23 @@ class Migration(migrations.Migration):
bases=(models.Model,),
),
migrations.CreateModel(
name='InstallationCount',
name="InstallationCount",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
('property', models.CharField(max_length=40)),
('end_time', models.DateTimeField()),
('interval', models.CharField(max_length=20)),
('value', models.BigIntegerField()),
("property", models.CharField(max_length=40)),
("end_time", models.DateTimeField()),
("interval", models.CharField(max_length=20)),
("value", models.BigIntegerField()),
(
'anomaly',
"anomaly",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to='analytics.Anomaly',
to="analytics.Anomaly",
null=True,
),
),
@@ -85,29 +85,29 @@ class Migration(migrations.Migration):
bases=(models.Model,),
),
migrations.CreateModel(
name='RealmCount',
name="RealmCount",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
(
'realm',
"realm",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'
on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
),
),
('property', models.CharField(max_length=40)),
('end_time', models.DateTimeField()),
('interval', models.CharField(max_length=20)),
('value', models.BigIntegerField()),
("property", models.CharField(max_length=40)),
("end_time", models.DateTimeField()),
("interval", models.CharField(max_length=20)),
("value", models.BigIntegerField()),
(
'anomaly',
"anomaly",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to='analytics.Anomaly',
to="analytics.Anomaly",
null=True,
),
),
@@ -115,35 +115,35 @@ class Migration(migrations.Migration):
bases=(models.Model,),
),
migrations.CreateModel(
name='StreamCount',
name="StreamCount",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
(
'realm',
"realm",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'
on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
),
),
(
'stream',
"stream",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream'
on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
),
),
('property', models.CharField(max_length=40)),
('end_time', models.DateTimeField()),
('interval', models.CharField(max_length=20)),
('value', models.BigIntegerField()),
("property", models.CharField(max_length=40)),
("end_time", models.DateTimeField()),
("interval", models.CharField(max_length=20)),
("value", models.BigIntegerField()),
(
'anomaly',
"anomaly",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to='analytics.Anomaly',
to="analytics.Anomaly",
null=True,
),
),
@@ -151,35 +151,35 @@ class Migration(migrations.Migration):
bases=(models.Model,),
),
migrations.CreateModel(
name='UserCount',
name="UserCount",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
(
'realm',
"realm",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'
on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
),
),
(
'user',
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
),
),
('property', models.CharField(max_length=40)),
('end_time', models.DateTimeField()),
('interval', models.CharField(max_length=20)),
('value', models.BigIntegerField()),
("property", models.CharField(max_length=40)),
("end_time", models.DateTimeField()),
("interval", models.CharField(max_length=20)),
("value", models.BigIntegerField()),
(
'anomaly',
"anomaly",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to='analytics.Anomaly',
to="analytics.Anomaly",
null=True,
),
),
@@ -187,23 +187,23 @@ class Migration(migrations.Migration):
bases=(models.Model,),
),
migrations.AlterUniqueTogether(
name='usercount',
unique_together={('user', 'property', 'end_time', 'interval')},
name="usercount",
unique_together={("user", "property", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='streamcount',
unique_together={('stream', 'property', 'end_time', 'interval')},
name="streamcount",
unique_together={("stream", "property", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='realmcount',
unique_together={('realm', 'property', 'end_time', 'interval')},
name="realmcount",
unique_together={("realm", "property", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='installationcount',
unique_together={('property', 'end_time', 'interval')},
name="installationcount",
unique_together={("property", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='huddlecount',
unique_together={('huddle', 'property', 'end_time', 'interval')},
name="huddlecount",
unique_together={("huddle", "property", "end_time", "interval")},
),
]

18
analytics/migrations/0002_remove_huddlecount.py
View File

@@ -4,27 +4,27 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0001_initial'),
("analytics", "0001_initial"),
]
operations = [
migrations.AlterUniqueTogether(
name='huddlecount',
name="huddlecount",
unique_together=set(),
),
migrations.RemoveField(
model_name='huddlecount',
name='anomaly',
model_name="huddlecount",
name="anomaly",
),
migrations.RemoveField(
model_name='huddlecount',
name='huddle',
model_name="huddlecount",
name="huddle",
),
migrations.RemoveField(
model_name='huddlecount',
name='user',
model_name="huddlecount",
name="user",
),
migrations.DeleteModel(
name='HuddleCount',
name="HuddleCount",
),
]

16
analytics/migrations/0003_fillstate.py
View File

@@ -4,23 +4,23 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0002_remove_huddlecount'),
("analytics", "0002_remove_huddlecount"),
]
operations = [
migrations.CreateModel(
name='FillState',
name="FillState",
fields=[
(
'id',
"id",
models.AutoField(
verbose_name='ID', serialize=False, auto_created=True, primary_key=True
verbose_name="ID", serialize=False, auto_created=True, primary_key=True
),
),
('property', models.CharField(unique=True, max_length=40)),
('end_time', models.DateTimeField()),
('state', models.PositiveSmallIntegerField()),
('last_modified', models.DateTimeField(auto_now=True)),
("property", models.CharField(unique=True, max_length=40)),
("end_time", models.DateTimeField()),
("state", models.PositiveSmallIntegerField()),
("last_modified", models.DateTimeField(auto_now=True)),
],
bases=(models.Model,),
),

18
analytics/migrations/0004_add_subgroup.py
View File

@@ -4,28 +4,28 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0003_fillstate'),
("analytics", "0003_fillstate"),
]
operations = [
migrations.AddField(
model_name='installationcount',
name='subgroup',
model_name="installationcount",
name="subgroup",
field=models.CharField(max_length=16, null=True),
),
migrations.AddField(
model_name='realmcount',
name='subgroup',
model_name="realmcount",
name="subgroup",
field=models.CharField(max_length=16, null=True),
),
migrations.AddField(
model_name='streamcount',
name='subgroup',
model_name="streamcount",
name="subgroup",
field=models.CharField(max_length=16, null=True),
),
migrations.AddField(
model_name='usercount',
name='subgroup',
model_name="usercount",
name="subgroup",
field=models.CharField(max_length=16, null=True),
),
]

34
analytics/migrations/0005_alter_field_size.py
View File

@@ -4,48 +4,48 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0004_add_subgroup'),
("analytics", "0004_add_subgroup"),
]
operations = [
migrations.AlterField(
model_name='installationcount',
name='interval',
model_name="installationcount",
name="interval",
field=models.CharField(max_length=8),
),
migrations.AlterField(
model_name='installationcount',
name='property',
model_name="installationcount",
name="property",
field=models.CharField(max_length=32),
),
migrations.AlterField(
model_name='realmcount',
name='interval',
model_name="realmcount",
name="interval",
field=models.CharField(max_length=8),
),
migrations.AlterField(
model_name='realmcount',
name='property',
model_name="realmcount",
name="property",
field=models.CharField(max_length=32),
),
migrations.AlterField(
model_name='streamcount',
name='interval',
model_name="streamcount",
name="interval",
field=models.CharField(max_length=8),
),
migrations.AlterField(
model_name='streamcount',
name='property',
model_name="streamcount",
name="property",
field=models.CharField(max_length=32),
),
migrations.AlterField(
model_name='usercount',
name='interval',
model_name="usercount",
name="interval",
field=models.CharField(max_length=8),
),
migrations.AlterField(
model_name='usercount',
name='property',
model_name="usercount",
name="property",
field=models.CharField(max_length=32),
),
]

18
analytics/migrations/0006_add_subgroup_to_unique_constraints.py
View File

@@ -4,24 +4,24 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0005_alter_field_size'),
("analytics", "0005_alter_field_size"),
]
operations = [
migrations.AlterUniqueTogether(
name='installationcount',
unique_together={('property', 'subgroup', 'end_time', 'interval')},
name="installationcount",
unique_together={("property", "subgroup", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='realmcount',
unique_together={('realm', 'property', 'subgroup', 'end_time', 'interval')},
name="realmcount",
unique_together={("realm", "property", "subgroup", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='streamcount',
unique_together={('stream', 'property', 'subgroup', 'end_time', 'interval')},
name="streamcount",
unique_together={("stream", "property", "subgroup", "end_time", "interval")},
),
migrations.AlterUniqueTogether(
name='usercount',
unique_together={('user', 'property', 'subgroup', 'end_time', 'interval')},
name="usercount",
unique_together={("user", "property", "subgroup", "end_time", "interval")},
),
]

34
analytics/migrations/0007_remove_interval.py
View File

@@ -5,40 +5,40 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0006_add_subgroup_to_unique_constraints'),
("analytics", "0006_add_subgroup_to_unique_constraints"),
]
operations = [
migrations.AlterUniqueTogether(
name='installationcount',
unique_together={('property', 'subgroup', 'end_time')},
name="installationcount",
unique_together={("property", "subgroup", "end_time")},
),
migrations.RemoveField(
model_name='installationcount',
name='interval',
model_name="installationcount",
name="interval",
),
migrations.AlterUniqueTogether(
name='realmcount',
unique_together={('realm', 'property', 'subgroup', 'end_time')},
name="realmcount",
unique_together={("realm", "property", "subgroup", "end_time")},
),
migrations.RemoveField(
model_name='realmcount',
name='interval',
model_name="realmcount",
name="interval",
),
migrations.AlterUniqueTogether(
name='streamcount',
unique_together={('stream', 'property', 'subgroup', 'end_time')},
name="streamcount",
unique_together={("stream", "property", "subgroup", "end_time")},
),
migrations.RemoveField(
model_name='streamcount',
name='interval',
model_name="streamcount",
name="interval",
),
migrations.AlterUniqueTogether(
name='usercount',
unique_together={('user', 'property', 'subgroup', 'end_time')},
name="usercount",
unique_together={("user", "property", "subgroup", "end_time")},
),
migrations.RemoveField(
model_name='usercount',
name='interval',
model_name="usercount",
name="interval",
),
]

16
analytics/migrations/0008_add_count_indexes.py
View File

@@ -5,21 +5,21 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('zerver', '0050_userprofile_avatar_version'),
('analytics', '0007_remove_interval'),
("zerver", "0050_userprofile_avatar_version"),
("analytics", "0007_remove_interval"),
]
operations = [
migrations.AlterIndexTogether(
name='realmcount',
index_together={('property', 'end_time')},
name="realmcount",
index_together={("property", "end_time")},
),
migrations.AlterIndexTogether(
name='streamcount',
index_together={('property', 'realm', 'end_time')},
name="streamcount",
index_together={("property", "realm", "end_time")},
),
migrations.AlterIndexTogether(
name='usercount',
index_together={('property', 'realm', 'end_time')},
name="usercount",
index_together={("property", "realm", "end_time")},
),
]

14
analytics/migrations/0009_remove_messages_to_stream_stat.py
View File

@@ -6,13 +6,13 @@ from django.db.migrations.state import StateApps
def delete_messages_sent_to_stream_stat(
apps: StateApps, schema_editor: DatabaseSchemaEditor
) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')
InstallationCount = apps.get_model('analytics', 'InstallationCount')
FillState = apps.get_model('analytics', 'FillState')
UserCount = apps.get_model("analytics", "UserCount")
StreamCount = apps.get_model("analytics", "StreamCount")
RealmCount = apps.get_model("analytics", "RealmCount")
InstallationCount = apps.get_model("analytics", "InstallationCount")
FillState = apps.get_model("analytics", "FillState")
property = 'messages_sent_to_stream:is_bot'
property = "messages_sent_to_stream:is_bot"
UserCount.objects.filter(property=property).delete()
StreamCount.objects.filter(property=property).delete()
RealmCount.objects.filter(property=property).delete()
@@ -23,7 +23,7 @@ def delete_messages_sent_to_stream_stat(
class Migration(migrations.Migration):
dependencies = [
('analytics', '0008_add_count_indexes'),
("analytics", "0008_add_count_indexes"),
]
operations = [

14
analytics/migrations/0010_clear_messages_sent_values.py
View File

@@ -6,13 +6,13 @@ from django.db.migrations.state import StateApps
def clear_message_sent_by_message_type_values(
apps: StateApps, schema_editor: DatabaseSchemaEditor
) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')
InstallationCount = apps.get_model('analytics', 'InstallationCount')
FillState = apps.get_model('analytics', 'FillState')
UserCount = apps.get_model("analytics", "UserCount")
StreamCount = apps.get_model("analytics", "StreamCount")
RealmCount = apps.get_model("analytics", "RealmCount")
InstallationCount = apps.get_model("analytics", "InstallationCount")
FillState = apps.get_model("analytics", "FillState")
property = 'messages_sent:message_type:day'
property = "messages_sent:message_type:day"
UserCount.objects.filter(property=property).delete()
StreamCount.objects.filter(property=property).delete()
RealmCount.objects.filter(property=property).delete()
@@ -22,7 +22,7 @@ def clear_message_sent_by_message_type_values(
class Migration(migrations.Migration):
dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
operations = [
migrations.RunPython(clear_message_sent_by_message_type_values),

12
analytics/migrations/0011_clear_analytics_tables.py
View File

@@ -4,11 +4,11 @@ from django.db.migrations.state import StateApps
def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')
InstallationCount = apps.get_model('analytics', 'InstallationCount')
FillState = apps.get_model('analytics', 'FillState')
UserCount = apps.get_model("analytics", "UserCount")
StreamCount = apps.get_model("analytics", "StreamCount")
RealmCount = apps.get_model("analytics", "RealmCount")
InstallationCount = apps.get_model("analytics", "InstallationCount")
FillState = apps.get_model("analytics", "FillState")
UserCount.objects.all().delete()
StreamCount.objects.all().delete()
@@ -20,7 +20,7 @@ def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor)
class Migration(migrations.Migration):
dependencies = [
('analytics', '0010_clear_messages_sent_values'),
("analytics", "0010_clear_messages_sent_values"),
]
operations = [

26
analytics/migrations/0012_add_on_delete.py
View File

@@ -7,36 +7,36 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0011_clear_analytics_tables'),
("analytics", "0011_clear_analytics_tables"),
]
operations = [
migrations.AlterField(
model_name='installationcount',
name='anomaly',
model_name="installationcount",
name="anomaly",
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'
null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
),
),
migrations.AlterField(
model_name='realmcount',
name='anomaly',
model_name="realmcount",
name="anomaly",
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'
null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
),
),
migrations.AlterField(
model_name='streamcount',
name='anomaly',
model_name="streamcount",
name="anomaly",
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'
null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
),
),
migrations.AlterField(
model_name='usercount',
name='anomaly',
model_name="usercount",
name="anomaly",
field=models.ForeignKey(
null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'
null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
),
),
]

20
analytics/migrations/0013_remove_anomaly.py
View File

@@ -6,27 +6,27 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0012_add_on_delete'),
("analytics", "0012_add_on_delete"),
]
operations = [
migrations.RemoveField(
model_name='installationcount',
name='anomaly',
model_name="installationcount",
name="anomaly",
),
migrations.RemoveField(
model_name='realmcount',
name='anomaly',
model_name="realmcount",
name="anomaly",
),
migrations.RemoveField(
model_name='streamcount',
name='anomaly',
model_name="streamcount",
name="anomaly",
),
migrations.RemoveField(
model_name='usercount',
name='anomaly',
model_name="usercount",
name="anomaly",
),
migrations.DeleteModel(
name='Anomaly',
name="Anomaly",
),
]

6
analytics/migrations/0014_remove_fillstate_last_modified.py
View File

@@ -6,12 +6,12 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('analytics', '0013_remove_anomaly'),
("analytics", "0013_remove_anomaly"),
]
operations = [
migrations.RemoveField(
model_name='fillstate',
name='last_modified',
model_name="fillstate",
name="last_modified",
),
]

24
analytics/migrations/0015_clear_duplicate_counts.py
View File

@@ -21,29 +21,29 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor)
additionally combine the sums.
"""
count_tables = dict(
realm=apps.get_model('analytics', 'RealmCount'),
user=apps.get_model('analytics', 'UserCount'),
stream=apps.get_model('analytics', 'StreamCount'),
installation=apps.get_model('analytics', 'InstallationCount'),
realm=apps.get_model("analytics", "RealmCount"),
user=apps.get_model("analytics", "UserCount"),
stream=apps.get_model("analytics", "StreamCount"),
installation=apps.get_model("analytics", "InstallationCount"),
)
for name, count_table in count_tables.items():
value = [name, 'property', 'end_time']
if name == 'installation':
value = ['property', 'end_time']
value = [name, "property", "end_time"]
if name == "installation":
value = ["property", "end_time"]
counts = (
count_table.objects.filter(subgroup=None)
.values(*value)
.annotate(Count('id'), Sum('value'))
.annotate(Count("id"), Sum("value"))
.filter(id__count__gt=1)
)
for count in counts:
count.pop('id__count')
total_value = count.pop('value__sum')
count.pop("id__count")
total_value = count.pop("value__sum")
duplicate_counts = list(count_table.objects.filter(**count))
first_count = duplicate_counts[0]
if count['property'] in ["invites_sent::day", "active_users_log:is_bot:day"]:
if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
# For LoggingCountStat objects, the right fix is to combine the totals;
# for other CountStat objects, we expect the duplicates to have the same value.
# And so all we need to do is delete them.
@@ -57,7 +57,7 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor)
class Migration(migrations.Migration):
dependencies = [
('analytics', '0014_remove_fillstate_last_modified'),
("analytics", "0014_remove_fillstate_last_modified"),
]
operations = [

58
analytics/migrations/0016_unique_constraint_when_subgroup_null.py
View File

@@ -6,88 +6,88 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('analytics', '0015_clear_duplicate_counts'),
("analytics", "0015_clear_duplicate_counts"),
]
operations = [
migrations.AlterUniqueTogether(
name='installationcount',
name="installationcount",
unique_together=set(),
),
migrations.AlterUniqueTogether(
name='realmcount',
name="realmcount",
unique_together=set(),
),
migrations.AlterUniqueTogether(
name='streamcount',
name="streamcount",
unique_together=set(),
),
migrations.AlterUniqueTogether(
name='usercount',
name="usercount",
unique_together=set(),
),
migrations.AddConstraint(
model_name='installationcount',
model_name="installationcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=False),
fields=('property', 'subgroup', 'end_time'),
name='unique_installation_count',
fields=("property", "subgroup", "end_time"),
name="unique_installation_count",
),
),
migrations.AddConstraint(
model_name='installationcount',
model_name="installationcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=True),
fields=('property', 'end_time'),
name='unique_installation_count_null_subgroup',
fields=("property", "end_time"),
name="unique_installation_count_null_subgroup",
),
),
migrations.AddConstraint(
model_name='realmcount',
model_name="realmcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=False),
fields=('realm', 'property', 'subgroup', 'end_time'),
name='unique_realm_count',
fields=("realm", "property", "subgroup", "end_time"),
name="unique_realm_count",
),
),
migrations.AddConstraint(
model_name='realmcount',
model_name="realmcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=True),
fields=('realm', 'property', 'end_time'),
name='unique_realm_count_null_subgroup',
fields=("realm", "property", "end_time"),
name="unique_realm_count_null_subgroup",
),
),
migrations.AddConstraint(
model_name='streamcount',
model_name="streamcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=False),
fields=('stream', 'property', 'subgroup', 'end_time'),
name='unique_stream_count',
fields=("stream", "property", "subgroup", "end_time"),
name="unique_stream_count",
),
),
migrations.AddConstraint(
model_name='streamcount',
model_name="streamcount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=True),
fields=('stream', 'property', 'end_time'),
name='unique_stream_count_null_subgroup',
fields=("stream", "property", "end_time"),
name="unique_stream_count_null_subgroup",
),
),
migrations.AddConstraint(
model_name='usercount',
model_name="usercount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=False),
fields=('user', 'property', 'subgroup', 'end_time'),
name='unique_user_count',
fields=("user", "property", "subgroup", "end_time"),
name="unique_user_count",
),
),
migrations.AddConstraint(
model_name='usercount',
model_name="usercount",
constraint=models.UniqueConstraint(
condition=models.Q(subgroup__isnull=True),
fields=('user', 'property', 'end_time'),
name='unique_user_count_null_subgroup',
fields=("user", "property", "end_time"),
name="unique_user_count_null_subgroup",
),
),
]

20
analytics/models.py
View File

@@ -24,8 +24,8 @@ class FillState(models.Model):
# The earliest/starting end_time in FillState
# We assume there is at least one realm
def installation_epoch() -> datetime.datetime:
earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))[
'date_created__min'
earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
"date_created__min"
]
return floor_to_day(earliest_realm_creation)
@@ -50,12 +50,12 @@ class InstallationCount(BaseCount):
UniqueConstraint(
fields=["property", "subgroup", "end_time"],
condition=Q(subgroup__isnull=False),
name='unique_installation_count',
name="unique_installation_count",
),
UniqueConstraint(
fields=["property", "end_time"],
condition=Q(subgroup__isnull=True),
name='unique_installation_count_null_subgroup',
name="unique_installation_count_null_subgroup",
),
]
@@ -72,12 +72,12 @@ class RealmCount(BaseCount):
UniqueConstraint(
fields=["realm", "property", "subgroup", "end_time"],
condition=Q(subgroup__isnull=False),
name='unique_realm_count',
name="unique_realm_count",
),
UniqueConstraint(
fields=["realm", "property", "end_time"],
condition=Q(subgroup__isnull=True),
name='unique_realm_count_null_subgroup',
name="unique_realm_count_null_subgroup",
),
]
index_together = ["property", "end_time"]
@@ -96,12 +96,12 @@ class UserCount(BaseCount):
UniqueConstraint(
fields=["user", "property", "subgroup", "end_time"],
condition=Q(subgroup__isnull=False),
name='unique_user_count',
name="unique_user_count",
),
UniqueConstraint(
fields=["user", "property", "end_time"],
condition=Q(subgroup__isnull=True),
name='unique_user_count_null_subgroup',
name="unique_user_count_null_subgroup",
),
]
# This index dramatically improves the performance of
@@ -122,12 +122,12 @@ class StreamCount(BaseCount):
UniqueConstraint(
fields=["stream", "property", "subgroup", "end_time"],
condition=Q(subgroup__isnull=False),
name='unique_stream_count',
name="unique_stream_count",
),
UniqueConstraint(
fields=["stream", "property", "end_time"],
condition=Q(subgroup__isnull=True),
name='unique_stream_count_null_subgroup',
name="unique_stream_count_null_subgroup",
),
]
# This index dramatically improves the performance of

674
analytics/tests/test_counts.py
View File

File diff suppressed because it is too large Load Diff

508
analytics/tests/test_views.py
View File

File diff suppressed because it is too large Load Diff

32
analytics/urls.py
View File

@@ -21,18 +21,18 @@ from zerver.lib.rest import rest_path
i18n_urlpatterns = [
# Server admin (user_profile.is_staff) visible stats pages
path('activity', get_activity),
path('activity/support', support, name='support'),
path('realm_activity/<realm_str>/', get_realm_activity),
path('user_activity/<email>/', get_user_activity),
path('stats/realm/<realm_str>/', stats_for_realm),
path('stats/installation', stats_for_installation),
path('stats/remote/<int:remote_server_id>/installation', stats_for_remote_installation),
path("activity", get_activity),
path("activity/support", support, name="support"),
path("realm_activity/<realm_str>/", get_realm_activity),
path("user_activity/<email>/", get_user_activity),
path("stats/realm/<realm_str>/", stats_for_realm),
path("stats/installation", stats_for_installation),
path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
path(
'stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/', stats_for_remote_realm
"stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
),
# User-visible stats page
path('stats', stats, name='stats'),
path("stats", stats, name="stats"),
]
# These endpoints are a part of the API (V1), which uses:
@@ -45,22 +45,22 @@ i18n_urlpatterns = [
# All of these paths are accessed by either a /json or /api prefix
v1_api_and_json_patterns = [
# get data for the graphs at /stats
rest_path('analytics/chart_data', GET=get_chart_data),
rest_path('analytics/chart_data/realm/<realm_str>', GET=get_chart_data_for_realm),
rest_path('analytics/chart_data/installation', GET=get_chart_data_for_installation),
rest_path("analytics/chart_data", GET=get_chart_data),
rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
rest_path(
'analytics/chart_data/remote/<int:remote_server_id>/installation',
"analytics/chart_data/remote/<int:remote_server_id>/installation",
GET=get_chart_data_for_remote_installation,
),
rest_path(
'analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>',
"analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
GET=get_chart_data_for_remote_realm,
),
]
i18n_urlpatterns += [
path('api/v1/', include(v1_api_and_json_patterns)),
path('json/', include(v1_api_and_json_patterns)),
path("api/v1/", include(v1_api_and_json_patterns)),
path("json/", include(v1_api_and_json_patterns)),
]
urlpatterns = i18n_urlpatterns

492
analytics/views.py
View File

File diff suppressed because it is too large Load Diff