paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-02 13:03:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

analytics: Use python 3 syntax for typing.

Browse Source
This commit is contained in:
rht
2017-11-05 06:54:00 +01:00
committed by Tim Abbott
parent b2637c8400
commit d1689b5884
18 changed files with 164 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
analytics/lib/time_utils.py
View File

@@ -8,8 +8,7 @@ from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
# If min_length is greater than 0, pads the list to the left.
# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
def time_range(start, end, frequency, min_length):
# type: (datetime, datetime, str, Optional[int]) -> List[datetime]
def time_range(start: datetime, end: datetime, frequency: str, min_length: Optional[int]) -> List[datetime]:
verify_UTC(start)
verify_UTC(end)
if frequency == CountStat.HOUR:

9
analytics/management/commands/analyze_mit.py
View File

@@ -8,8 +8,7 @@ from django.core.management.base import BaseCommand, CommandParser
from zerver.lib.timestamp import timestamp_to_datetime
from zerver.models import Message, Recipient
def compute_stats(log_level):
# type: (int) -> None
def compute_stats(log_level: int) -> None:
logger = logging.getLogger()
logger.setLevel(log_level)
@@ -72,12 +71,10 @@ def compute_stats(log_level):
class Command(BaseCommand):
help = "Compute statistics on MIT Zephyr usage."
def add_arguments(self, parser):
# type: (CommandParser) -> None
def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument('--verbose', default=False, action='store_true')
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
level = logging.INFO
if options["verbose"]:
level = logging.DEBUG

9
analytics/management/commands/analyze_user_activity.py
View File

@@ -7,8 +7,7 @@ from django.utils.timezone import utc
from zerver.lib.statistics import seconds_usage_between
from zerver.models import UserProfile
def analyze_activity(options):
# type: (Dict[str, Any]) -> None
def analyze_activity(options: Dict[str, Any]) -> None:
day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
day_end = day_start + datetime.timedelta(days=options["duration"])
@@ -47,13 +46,11 @@ Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--
By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
is shown for all realms"""
def add_arguments(self, parser):
# type: (CommandParser) -> None
def add_arguments(self, parser: CommandParser) -> None:
parser.add_argument('--realm', action='store')
parser.add_argument('--date', action='store', default="2013-09-06")
parser.add_argument('--duration', action='store', default=1, type=int,
help="How many days to show usage information for")
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
analyze_activity(options)

6
analytics/management/commands/clear_analytics_tables.py
View File

@@ -9,14 +9,12 @@ from analytics.lib.counts import do_drop_all_analytics_tables
class Command(BaseCommand):
help = """Clear analytics tables."""
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--force',
action='store_true',
help="Clear analytics tables.")
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
if options['force']:
do_drop_all_analytics_tables()
else:

6
analytics/management/commands/clear_single_stat.py
View File

@@ -9,8 +9,7 @@ from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
class Command(BaseCommand):
help = """Clear analytics tables."""
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--force',
action='store_true',
help="Actually do it.")
@@ -18,8 +17,7 @@ class Command(BaseCommand):
type=str,
help="The property of the stat to be cleared.")
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
property = options['property']
if property not in COUNT_STATS:
print("Invalid property: %s" % (property,))

9
analytics/management/commands/client_activity.py
View File

@@ -17,8 +17,7 @@ Usage examples:
./manage.py client_activity --target realm --realm zulip
./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--target', dest='target', required=True, type=str,
help="'server' will calculate client activity of the entire server. "
"'realm' will calculate client activity of realm. "
@@ -27,8 +26,7 @@ Usage examples:
help="The email address of the user you want to calculate activity.")
self.add_realm_args(parser)
def compute_activity(self, user_activity_objects):
# type: (QuerySet) -> None
def compute_activity(self, user_activity_objects: QuerySet) -> None:
# Report data from the past week.
#
# This is a rough report of client activity because we inconsistently
@@ -58,8 +56,7 @@ Usage examples:
print("%25s %15d" % (count[1], count[0]))
print("Total:", total)
def handle(self, *args, **options):
# type: (*Any, **str) -> None
def handle(self, *args: Any, **options: str) -> None:
realm = self.get_realm(options)
if options["user"] is None:
if options["target"] == "server" and realm is None:

15
analytics/management/commands/populate_analytics_db.py
View File

@@ -19,8 +19,11 @@ class Command(BaseCommand):
DAYS_OF_DATA = 100
random_seed = 26
def create_user(self, email, full_name, is_staff, date_joined, realm):
# type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
def create_user(self, email: Text,
full_name: Text,
is_staff: bool,
date_joined: datetime,
realm: Realm) -> UserProfile:
user = UserProfile.objects.create(
email=email, full_name=full_name, is_staff=is_staff,
realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
@@ -41,8 +44,7 @@ class Command(BaseCommand):
autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
do_drop_all_analytics_tables()
# I believe this also deletes any objects with this realm as a foreign key
Realm.objects.filter(string_id='analytics').delete()
@@ -53,8 +55,9 @@ class Command(BaseCommand):
string_id='analytics', name='Analytics', date_created=installation_time)
shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
def insert_fixture_data(stat, fixture_data, table):
# type: (CountStat, Mapping[Optional[str], List[int]], Type[BaseCount]) -> None
def insert_fixture_data(stat: CountStat,
fixture_data: Mapping[Optional[str], List[int]],
table: Type[BaseCount]) -> None:
end_times = time_range(last_end_time, last_end_time, stat.frequency,
len(list(fixture_data.values())[0]))
if table == RealmCount:

33
analytics/management/commands/realm_stats.py
View File

@@ -18,13 +18,11 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
class Command(BaseCommand):
help = "Generate statistics on realm activity."
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
help="realm to generate statistics for")
def active_users(self, realm):
# type: (Realm) -> List[UserProfile]
def active_users(self, realm: Realm) -> List[UserProfile]:
# Has been active (on the website, for now) in the last 7 days.
activity_cutoff = timezone_now() - datetime.timedelta(days=7)
return [activity.user_profile for activity in (
@@ -34,53 +32,44 @@ class Command(BaseCommand):
query="/json/users/me/pointer",
client__name="website"))]
def messages_sent_by(self, user, days_ago):
# type: (UserProfile, int) -> int
def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
def total_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def total_messages(self, realm: Realm, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
def human_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def human_messages(self, realm: Realm, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
def api_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def api_messages(self, realm: Realm, days_ago: int) -> int:
return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
def stream_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def stream_messages(self, realm: Realm, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
recipient__type=Recipient.STREAM).count()
def private_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def private_messages(self, realm: Realm, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
def group_private_messages(self, realm, days_ago):
# type: (Realm, int) -> int
def group_private_messages(self, realm: Realm, days_ago: int) -> int:
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
def report_percentage(self, numerator, denominator, text):
# type: (float, float, str) -> None
def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
if not denominator:
fraction = 0.0
else:
fraction = numerator / float(denominator)
print("%.2f%% of" % (fraction * 100,), text)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
if options['realms']:
try:
realms = [get_realm(string_id) for string_id in options['realms']]

6
analytics/management/commands/stream_stats.py
View File

@@ -10,13 +10,11 @@ from zerver.models import Message, Realm, \
class Command(BaseCommand):
help = "Generate statistics on the streams for a realm."
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
help="realm to generate statistics for")
def handle(self, *args, **options):
# type: (*Any, **str) -> None
def handle(self, *args: Any, **options: str) -> None:
if options['realms']:
try:
realms = [get_realm(string_id) for string_id in options['realms']]

9
analytics/management/commands/update_analytics_counts.py
View File

@@ -19,8 +19,7 @@ class Command(BaseCommand):
Run as a cron job that runs every hour."""
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('--time', '-t',
type=str,
help='Update stat tables from current state to'
@@ -38,8 +37,7 @@ class Command(BaseCommand):
help="Print timing information to stdout.",
default=False)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
try:
os.mkdir(settings.ANALYTICS_LOCK_DIR)
except OSError:
@@ -51,8 +49,7 @@ class Command(BaseCommand):
finally:
os.rmdir(settings.ANALYTICS_LOCK_DIR)
def run_update_analytics_counts(self, options):
# type: (Dict[str, Any]) -> None
def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
# installation_epoch relies on there being at least one realm; we
# shouldn't run the analytics code if that condition isn't satisfied
if not Realm.objects.exists():

9
analytics/management/commands/user_stats.py
View File

@@ -10,19 +10,16 @@ from zerver.models import Message, Realm, Stream, UserProfile, get_realm
class Command(BaseCommand):
help = "Generate statistics on user activity."
def add_arguments(self, parser):
# type: (ArgumentParser) -> None
def add_arguments(self, parser: ArgumentParser) -> None:
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
help="realm to generate statistics for")
def messages_sent_by(self, user, week):
# type: (UserProfile, int) -> int
def messages_sent_by(self, user: UserProfile, week: int) -> int:
start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
end = timezone_now() - datetime.timedelta(days=week*7)
return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
def handle(self, *args: Any, **options: Any) -> None:
if options['realms']:
try:
realms = [get_realm(string_id) for string_id in options['realms']]

3
analytics/migrations/0009_remove_messages_to_stream_stat.py
View File

@@ -3,8 +3,7 @@ from django.db import migrations
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
from django.db.migrations.state import StateApps
def delete_messages_sent_to_stream_stat(apps, schema_editor):
# type: (StateApps, DatabaseSchemaEditor) -> None
def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')

3
analytics/migrations/0010_clear_messages_sent_values.py
View File

@@ -3,8 +3,7 @@ from django.db import migrations
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
from django.db.migrations.state import StateApps
def clear_message_sent_by_message_type_values(apps, schema_editor):
# type: (StateApps, DatabaseSchemaEditor) -> None
def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')

3
analytics/migrations/0011_clear_analytics_tables.py
View File

@@ -3,8 +3,7 @@ from django.db import migrations
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
from django.db.migrations.state import StateApps
def clear_analytics_tables(apps, schema_editor):
# type: (StateApps, DatabaseSchemaEditor) -> None
def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
UserCount = apps.get_model('analytics', 'UserCount')
StreamCount = apps.get_model('analytics', 'StreamCount')
RealmCount = apps.get_model('analytics', 'RealmCount')

162
analytics/tests/test_counts.py
View File

@@ -32,8 +32,7 @@ class AnalyticsTestCase(TestCase):
TIME_ZERO = datetime(1988, 3, 14).replace(tzinfo=timezone_utc)
TIME_LAST_HOUR = TIME_ZERO - HOUR
def setUp(self):
# type: () -> None
def setUp(self) -> None:
self.default_realm = Realm.objects.create(
string_id='realmtest', name='Realm Test', date_created=self.TIME_ZERO - 2*self.DAY)
# used to generate unique names in self.create_*
@@ -42,8 +41,7 @@ class AnalyticsTestCase(TestCase):
self.current_property = None # type: Optional[str]
# Lightweight creation of users, streams, and messages
def create_user(self, **kwargs):
# type: (**Any) -> UserProfile
def create_user(self, **kwargs: Any) -> UserProfile:
self.name_counter += 1
defaults = {
'email': 'user%s@domain.tld' % (self.name_counter,),
@@ -58,8 +56,7 @@ class AnalyticsTestCase(TestCase):
kwargs[key] = kwargs.get(key, value)
return UserProfile.objects.create(**kwargs)
def create_stream_with_recipient(self, **kwargs):
# type: (**Any) -> Tuple[Stream, Recipient]
def create_stream_with_recipient(self, **kwargs: Any) -> Tuple[Stream, Recipient]:
self.name_counter += 1
defaults = {'name': 'stream name %s' % (self.name_counter,),
'realm': self.default_realm,
@@ -70,8 +67,7 @@ class AnalyticsTestCase(TestCase):
recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
return stream, recipient
def create_huddle_with_recipient(self, **kwargs):
# type: (**Any) -> Tuple[Huddle, Recipient]
def create_huddle_with_recipient(self, **kwargs: Any) -> Tuple[Huddle, Recipient]:
self.name_counter += 1
defaults = {'huddle_hash': 'hash%s' % (self.name_counter,)}
for key, value in defaults.items():
@@ -80,8 +76,7 @@ class AnalyticsTestCase(TestCase):
recipient = Recipient.objects.create(type_id=huddle.id, type=Recipient.HUDDLE)
return huddle, recipient
def create_message(self, sender, recipient, **kwargs):
# type: (UserProfile, Recipient, **Any) -> Message
def create_message(self, sender: UserProfile, recipient: Recipient, **kwargs: Any) -> Message:
defaults = {
'sender': sender,
'recipient': recipient,
@@ -108,8 +103,7 @@ class AnalyticsTestCase(TestCase):
queryset = queryset.filter(subgroup=subgroup)
self.assertEqual(queryset.values_list('value', flat=True)[0], value)
def assertTableState(self, table, arg_keys, arg_values):
# type: (Type[BaseCount], List[str], List[List[Union[int, str, bool, datetime, Realm, UserProfile, Stream]]]) -> None
def assertTableState(self, table: Type[BaseCount], arg_keys: List[str], arg_values: List[List[object]]) -> None:
"""Assert that the state of a *Count table is what it should be.
Example usage:
@@ -154,20 +148,17 @@ class AnalyticsTestCase(TestCase):
self.assertEqual(table.objects.count(), len(arg_values))
class TestProcessCountStat(AnalyticsTestCase):
def make_dummy_count_stat(self, property):
# type: (str) -> CountStat
def make_dummy_count_stat(self, property: str) -> CountStat:
query = """INSERT INTO analytics_realmcount (realm_id, value, property, end_time)
VALUES (%s, 1, '%s', %%%%(time_end)s)""" % (self.default_realm.id, property)
return CountStat(property, sql_data_collector(RealmCount, query, None), CountStat.HOUR)
def assertFillStateEquals(self, stat, end_time, state=FillState.DONE):
# type: (CountStat, datetime, int) -> None
def assertFillStateEquals(self, stat: CountStat, end_time: datetime, state: int=FillState.DONE) -> None:
fill_state = FillState.objects.filter(property=stat.property).first()
self.assertEqual(fill_state.end_time, end_time)
self.assertEqual(fill_state.state, state)
def test_process_stat(self):
# type: () -> None
def test_process_stat(self) -> None:
# process new stat
current_time = installation_epoch() + self.HOUR
stat = self.make_dummy_count_stat('test stat')
@@ -193,8 +184,7 @@ class TestProcessCountStat(AnalyticsTestCase):
self.assertFillStateEquals(stat, current_time)
self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 2)
def test_bad_fill_to_time(self):
# type: () -> None
def test_bad_fill_to_time(self) -> None:
stat = self.make_dummy_count_stat('test stat')
with self.assertRaises(ValueError):
process_count_stat(stat, installation_epoch() + 65*self.MINUTE)
@@ -204,8 +194,7 @@ class TestProcessCountStat(AnalyticsTestCase):
# This tests the LoggingCountStat branch of the code in do_delete_counts_at_hour.
# It is important that do_delete_counts_at_hour not delete any of the collected
# logging data!
def test_process_logging_stat(self):
# type: () -> None
def test_process_logging_stat(self) -> None:
end_time = self.TIME_ZERO
user_stat = LoggingCountStat('user stat', UserCount, CountStat.DAY)
@@ -247,8 +236,7 @@ class TestProcessCountStat(AnalyticsTestCase):
self.assertTableState(InstallationCount, ['property', 'value'],
[[user_stat.property, 6], [stream_stat.property, 6], [realm_stat.property, 6]])
def test_process_dependent_stat(self):
# type: () -> None
def test_process_dependent_stat(self) -> None:
stat1 = self.make_dummy_count_stat('stat1')
stat2 = self.make_dummy_count_stat('stat2')
query = """INSERT INTO analytics_realmcount (realm_id, value, property, end_time)
@@ -297,8 +285,7 @@ class TestProcessCountStat(AnalyticsTestCase):
self.assertFillStateEquals(stat4, hour24)
class TestCountStats(AnalyticsTestCase):
def setUp(self):
# type: () -> None
def setUp(self) -> None:
super().setUp()
# This tests two things for each of the queries/CountStats: Handling
# more than 1 realm, and the time bounds (time_start and time_end in
@@ -327,8 +314,7 @@ class TestCountStats(AnalyticsTestCase):
# This huddle should not show up anywhere
self.create_huddle_with_recipient()
def test_active_users_by_is_bot(self):
# type: () -> None
def test_active_users_by_is_bot(self) -> None:
stat = COUNT_STATS['active_users:is_bot:day']
self.current_property = stat.property
@@ -350,8 +336,7 @@ class TestCountStats(AnalyticsTestCase):
self.assertTableState(UserCount, [], [])
self.assertTableState(StreamCount, [], [])
def test_messages_sent_by_is_bot(self):
# type: () -> None
def test_messages_sent_by_is_bot(self) -> None:
stat = COUNT_STATS['messages_sent:is_bot:hour']
self.current_property = stat.property
@@ -379,8 +364,7 @@ class TestCountStats(AnalyticsTestCase):
self.assertTableState(InstallationCount, ['value', 'subgroup'], [[3, 'false'], [3, 'true']])
self.assertTableState(StreamCount, [], [])
def test_messages_sent_by_message_type(self):
# type: () -> None
def test_messages_sent_by_message_type(self) -> None:
stat = COUNT_STATS['messages_sent:message_type:day']
self.current_property = stat.property
@@ -442,8 +426,7 @@ class TestCountStats(AnalyticsTestCase):
[2, 'huddle_message']])
self.assertTableState(StreamCount, [], [])
def test_messages_sent_to_recipients_with_same_id(self):
# type: () -> None
def test_messages_sent_to_recipients_with_same_id(self) -> None:
stat = COUNT_STATS['messages_sent:message_type:day']
self.current_property = stat.property
@@ -462,8 +445,7 @@ class TestCountStats(AnalyticsTestCase):
self.assertCountEquals(UserCount, 1, subgroup='huddle_message')
self.assertCountEquals(UserCount, 1, subgroup='public_stream')
def test_messages_sent_by_client(self):
# type: () -> None
def test_messages_sent_by_client(self) -> None:
stat = COUNT_STATS['messages_sent:client:day']
self.current_property = stat.property
@@ -498,8 +480,7 @@ class TestCountStats(AnalyticsTestCase):
[[4, website_client_id], [3, client2_id]])
self.assertTableState(StreamCount, [], [])
def test_messages_sent_to_stream_by_is_bot(self):
# type: () -> None
def test_messages_sent_to_stream_by_is_bot(self) -> None:
stat = COUNT_STATS['messages_in_stream:is_bot:day']
self.current_property = stat.property
@@ -536,14 +517,12 @@ class TestCountStats(AnalyticsTestCase):
self.assertTableState(InstallationCount, ['value', 'subgroup'], [[5, 'false'], [2, 'true']])
self.assertTableState(UserCount, [], [])
def create_interval(self, user, start_offset, end_offset):
# type: (UserProfile, timedelta, timedelta) -> None
def create_interval(self, user: UserProfile, start_offset: timedelta, end_offset: timedelta) -> None:
UserActivityInterval.objects.create(
user_profile=user, start=self.TIME_ZERO-start_offset,
end=self.TIME_ZERO-end_offset)
def test_15day_actives(self):
# type: () -> None
def test_15day_actives(self) -> None:
stat = COUNT_STATS['15day_actives::day']
self.current_property = stat.property
@@ -586,8 +565,7 @@ class TestCountStats(AnalyticsTestCase):
self.assertTableState(InstallationCount, ['value'], [[6]])
self.assertTableState(StreamCount, [], [])
def test_minutes_active(self):
# type: () -> None
def test_minutes_active(self) -> None:
stat = COUNT_STATS['minutes_active::day']
self.current_property = stat.property
@@ -635,16 +613,14 @@ class TestDoAggregateToSummaryTable(AnalyticsTestCase):
# feature important for keeping the size of the analytics tables small,
# which is that if there is no relevant data in the table being
# aggregated, the aggregation table doesn't get a row with value 0.
def test_no_aggregated_zeros(self):
# type: () -> None
def test_no_aggregated_zeros(self) -> None:
stat = LoggingCountStat('test stat', UserCount, CountStat.HOUR)
do_aggregate_to_summary_table(stat, self.TIME_ZERO)
self.assertFalse(RealmCount.objects.exists())
self.assertFalse(InstallationCount.objects.exists())
class TestDoIncrementLoggingStat(AnalyticsTestCase):
def test_table_and_id_args(self):
# type: () -> None
def test_table_and_id_args(self) -> None:
# For realms, streams, and users, tests that the new rows are going to
# the appropriate *Count table, and that using a different zerver_object
# results in a new row being created
@@ -669,8 +645,7 @@ class TestDoIncrementLoggingStat(AnalyticsTestCase):
do_increment_logging_stat(stream2, stat, None, self.TIME_ZERO)
self.assertTableState(StreamCount, ['stream'], [[stream1], [stream2]])
def test_frequency(self):
# type: () -> None
def test_frequency(self) -> None:
times = [self.TIME_ZERO - self.MINUTE*i for i in [0, 1, 61, 24*60+1]]
stat = LoggingCountStat('day test', RealmCount, CountStat.DAY)
@@ -687,8 +662,7 @@ class TestDoIncrementLoggingStat(AnalyticsTestCase):
[1, 'hour test', self.TIME_LAST_HOUR],
[1, 'hour test', self.TIME_ZERO - self.DAY]])
def test_get_or_create(self):
# type: () -> None
def test_get_or_create(self) -> None:
stat = LoggingCountStat('test', RealmCount, CountStat.HOUR)
# All these should trigger the create part of get_or_create.
# property is tested in test_frequency, and id_args are tested in test_id_args,
@@ -706,8 +680,7 @@ class TestDoIncrementLoggingStat(AnalyticsTestCase):
[[2, 'subgroup1', self.TIME_ZERO], [1, 'subgroup2', self.TIME_ZERO],
[1, 'subgroup1', self.TIME_LAST_HOUR]])
def test_increment(self):
# type: () -> None
def test_increment(self) -> None:
stat = LoggingCountStat('test', RealmCount, CountStat.DAY)
self.current_property = 'test'
do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO, increment=-1)
@@ -718,8 +691,7 @@ class TestDoIncrementLoggingStat(AnalyticsTestCase):
self.assertTableState(RealmCount, ['value'], [[3]])
class TestLoggingCountStats(AnalyticsTestCase):
def test_aggregation(self):
# type: () -> None
def test_aggregation(self) -> None:
stat = LoggingCountStat('realm test', RealmCount, CountStat.DAY)
do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
process_count_stat(stat, self.TIME_ZERO)
@@ -741,8 +713,7 @@ class TestLoggingCountStats(AnalyticsTestCase):
self.assertTableState(UserCount, ['property', 'value'], [['user test', 1]])
self.assertTableState(StreamCount, ['property', 'value'], [['stream test', 1]])
def test_active_users_log_by_is_bot(self):
# type: () -> None
def test_active_users_log_by_is_bot(self) -> None:
property = 'active_users_log:is_bot:day'
user = do_create_user('email', 'password', self.default_realm, 'full_name', 'short_name')
self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
@@ -761,8 +732,7 @@ class TestLoggingCountStats(AnalyticsTestCase):
.aggregate(Sum('value'))['value__sum'])
class TestDeleteStats(AnalyticsTestCase):
def test_do_drop_all_analytics_tables(self):
# type: () -> None
def test_do_drop_all_analytics_tables(self) -> None:
user = self.create_user()
stream = self.create_stream_with_recipient()[0]
count_args = {'property': 'test', 'end_time': self.TIME_ZERO, 'value': 10}
@@ -782,8 +752,7 @@ class TestDeleteStats(AnalyticsTestCase):
for table in list(analytics.models.values()):
self.assertFalse(table.objects.exists())
def test_do_drop_single_stat(self):
# type: () -> None
def test_do_drop_single_stat(self) -> None:
user = self.create_user()
stream = self.create_stream_with_recipient()[0]
count_args_to_delete = {'property': 'to_delete', 'end_time': self.TIME_ZERO, 'value': 10}
@@ -811,15 +780,13 @@ class TestDeleteStats(AnalyticsTestCase):
self.assertTrue(table.objects.filter(property='to_save').exists())
class TestActiveUsersAudit(AnalyticsTestCase):
def setUp(self):
# type: () -> None
def setUp(self) -> None:
super().setUp()
self.user = self.create_user()
self.stat = COUNT_STATS['active_users_audit:is_bot:day']
self.current_property = self.stat.property
def add_event(self, event_type, days_offset, user=None):
# type: (str, float, Optional[UserProfile]) -> None
def add_event(self, event_type: str, days_offset: float, user: Optional[UserProfile]=None) -> None:
hours_offset = int(24*days_offset)
if user is None:
user = self.user
@@ -827,29 +794,25 @@ class TestActiveUsersAudit(AnalyticsTestCase):
realm=user.realm, modified_user=user, event_type=event_type,
event_time=self.TIME_ZERO - hours_offset*self.HOUR)
def test_user_deactivated_in_future(self):
# type: () -> None
def test_user_deactivated_in_future(self) -> None:
self.add_event('user_created', 1)
self.add_event('user_deactivated', 0)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup'], [['false']])
def test_user_reactivated_in_future(self):
# type: () -> None
def test_user_reactivated_in_future(self) -> None:
self.add_event('user_deactivated', 1)
self.add_event('user_reactivated', 0)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, [], [])
def test_user_active_then_deactivated_same_day(self):
# type: () -> None
def test_user_active_then_deactivated_same_day(self) -> None:
self.add_event('user_created', 1)
self.add_event('user_deactivated', .5)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, [], [])
def test_user_unactive_then_activated_same_day(self):
# type: () -> None
def test_user_unactive_then_activated_same_day(self) -> None:
self.add_event('user_deactivated', 1)
self.add_event('user_reactivated', .5)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
@@ -857,23 +820,20 @@ class TestActiveUsersAudit(AnalyticsTestCase):
# Arguably these next two tests are duplicates of the _in_future tests, but are
# a guard against future refactorings where they may no longer be duplicates
def test_user_active_then_deactivated_with_day_gap(self):
# type: () -> None
def test_user_active_then_deactivated_with_day_gap(self) -> None:
self.add_event('user_created', 2)
self.add_event('user_deactivated', 1)
process_count_stat(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup', 'end_time'],
[['false', self.TIME_ZERO - self.DAY]])
def test_user_deactivated_then_reactivated_with_day_gap(self):
# type: () -> None
def test_user_deactivated_then_reactivated_with_day_gap(self) -> None:
self.add_event('user_deactivated', 2)
self.add_event('user_reactivated', 1)
process_count_stat(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup'], [['false']])
def test_event_types(self):
# type: () -> None
def test_event_types(self) -> None:
self.add_event('user_created', 4)
self.add_event('user_deactivated', 3)
self.add_event('user_activated', 2)
@@ -885,8 +845,7 @@ class TestActiveUsersAudit(AnalyticsTestCase):
# Also tests that aggregation to RealmCount and InstallationCount is
# being done, and that we're storing the user correctly in UserCount
def test_multiple_users_realms_and_bots(self):
# type: () -> None
def test_multiple_users_realms_and_bots(self) -> None:
user1 = self.create_user()
user2 = self.create_user()
second_realm = Realm.objects.create(string_id='moo', name='moo')
@@ -910,8 +869,7 @@ class TestActiveUsersAudit(AnalyticsTestCase):
# do_fill_count_stat_at_hour. E.g. if one changes self.stat.frequency to
# CountStat.HOUR from CountStat.DAY, this will fail, while many of the
# tests above will not.
def test_update_from_two_days_ago(self):
# type: () -> None
def test_update_from_two_days_ago(self) -> None:
self.add_event('user_created', 2)
process_count_stat(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup', 'end_time'],
@@ -920,31 +878,27 @@ class TestActiveUsersAudit(AnalyticsTestCase):
# User with no relevant activity could happen e.g. for a system bot that
# doesn't go through do_create_user. Mainly just want to make sure that
# that situation doesn't throw an error.
def test_empty_realm_or_user_with_no_relevant_activity(self):
# type: () -> None
def test_empty_realm_or_user_with_no_relevant_activity(self) -> None:
self.add_event('unrelated', 1)
self.create_user() # also test a user with no RealmAuditLog entries
Realm.objects.create(string_id='moo', name='moo')
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, [], [])
def test_max_audit_entry_is_unrelated(self):
# type: () -> None
def test_max_audit_entry_is_unrelated(self) -> None:
self.add_event('user_created', 1)
self.add_event('unrelated', .5)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup'], [['false']])
# Simultaneous related audit entries should not be allowed, and so not testing for that.
def test_simultaneous_unrelated_audit_entry(self):
# type: () -> None
def test_simultaneous_unrelated_audit_entry(self) -> None:
self.add_event('user_created', 1)
self.add_event('unrelated', 1)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(UserCount, ['subgroup'], [['false']])
def test_simultaneous_max_audit_entries_of_different_users(self):
# type: () -> None
def test_simultaneous_max_audit_entries_of_different_users(self) -> None:
user1 = self.create_user()
user2 = self.create_user()
user3 = self.create_user()
@@ -956,8 +910,7 @@ class TestActiveUsersAudit(AnalyticsTestCase):
self.assertTableState(UserCount, ['user', 'subgroup'],
[[user1, 'false'], [user2, 'false']])
def test_end_to_end_with_actions_dot_py(self):
# type: () -> None
def test_end_to_end_with_actions_dot_py(self) -> None:
user1 = do_create_user('email1', 'password', self.default_realm, 'full_name', 'short_name')
user2 = do_create_user('email2', 'password', self.default_realm, 'full_name', 'short_name')
user3 = do_create_user('email3', 'password', self.default_realm, 'full_name', 'short_name')
@@ -974,30 +927,26 @@ class TestActiveUsersAudit(AnalyticsTestCase):
self.assertFalse(UserCount.objects.filter(user=user2).exists())
class TestRealmActiveHumans(AnalyticsTestCase):
def setUp(self):
# type: () -> None
def setUp(self) -> None:
super().setUp()
self.stat = COUNT_STATS['realm_active_humans::day']
self.current_property = self.stat.property
def mark_audit_active(self, user, end_time=None):
# type: (UserProfile, Optional[datetime]) -> None
def mark_audit_active(self, user: UserProfile, end_time: Optional[datetime]=None) -> None:
if end_time is None:
end_time = self.TIME_ZERO
UserCount.objects.create(
user=user, realm=user.realm, property='active_users_audit:is_bot:day',
subgroup=ujson.dumps(user.is_bot), end_time=end_time, value=1)
def mark_15day_active(self, user, end_time=None):
# type: (UserProfile, Optional[datetime]) -> None
def mark_15day_active(self, user: UserProfile, end_time: Optional[datetime]=None) -> None:
if end_time is None:
end_time = self.TIME_ZERO
UserCount.objects.create(
user=user, realm=user.realm, property='15day_actives::day',
end_time=end_time, value=1)
def test_basic_boolean_logic(self):
# type: () -> None
def test_basic_boolean_logic(self) -> None:
user = self.create_user()
self.mark_audit_active(user, end_time=self.TIME_ZERO - self.DAY)
self.mark_15day_active(user, end_time=self.TIME_ZERO)
@@ -1008,16 +957,14 @@ class TestRealmActiveHumans(AnalyticsTestCase):
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i*self.DAY)
self.assertTableState(RealmCount, ['value', 'end_time'], [[1, self.TIME_ZERO + self.DAY]])
def test_bots_not_counted(self):
# type: () -> None
def test_bots_not_counted(self) -> None:
bot = self.create_user(is_bot=True)
self.mark_audit_active(bot)
self.mark_15day_active(bot)
do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
self.assertTableState(RealmCount, [], [])
def test_multiple_users_realms_and_times(self):
# type: () -> None
def test_multiple_users_realms_and_times(self) -> None:
user1 = self.create_user()
user2 = self.create_user()
second_realm = Realm.objects.create(string_id='second', name='second')
@@ -1057,8 +1004,7 @@ class TestRealmActiveHumans(AnalyticsTestCase):
[1, self.default_realm, self.TIME_ZERO - self.DAY],
[2, second_realm, self.TIME_ZERO - self.DAY]])
def test_end_to_end(self):
# type: () -> None
def test_end_to_end(self) -> None:
user1 = do_create_user('email1', 'password', self.default_realm, 'full_name', 'short_name')
user2 = do_create_user('email2', 'password', self.default_realm, 'full_name', 'short_name')
do_create_user('email3', 'password', self.default_realm, 'full_name', 'short_name')

3
analytics/tests/test_fixtures.py
View File

@@ -4,8 +4,7 @@ from zerver.lib.test_classes import ZulipTestCase
# A very light test suite; the code being tested is not run in production.
class TestFixtures(ZulipTestCase):
def test_deterministic_settings(self):
# type: () -> None
def test_deterministic_settings(self) -> None:
# test basic business_hour / non_business_hour calculation
# test we get an array of the right length with frequency=CountStat.DAY
data = generate_time_series_data(

54
analytics/tests/test_views.py
View File

@@ -16,8 +16,7 @@ from zerver.lib.timestamp import ceiling_to_day, \
from zerver.models import Client, get_realm
class TestStatsEndpoint(ZulipTestCase):
def test_stats(self):
# type: () -> None
def test_stats(self) -> None:
self.user = self.example_user('hamlet')
self.login(self.user.email)
result = self.client_get('/stats')
@@ -26,8 +25,7 @@ class TestStatsEndpoint(ZulipTestCase):
self.assert_in_response("Zulip analytics for", result)
class TestGetChartData(ZulipTestCase):
def setUp(self):
# type: () -> None
def setUp(self) -> None:
self.realm = get_realm('zulip')
self.user = self.example_user('hamlet')
self.login(self.user.email)
@@ -36,12 +34,10 @@ class TestGetChartData(ZulipTestCase):
self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
for i in range(4)]
def data(self, i):
# type: (int) -> List[int]
def data(self, i: int) -> List[int]:
return [0, 0, i, 0]
def insert_data(self, stat, realm_subgroups, user_subgroups):
# type: (CountStat, List[Optional[str]], List[str]) -> None
def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]], user_subgroups: List[str]) -> None:
if stat.frequency == CountStat.HOUR:
insert_time = self.end_times_hour[2]
fill_time = self.end_times_hour[-1]
@@ -59,8 +55,7 @@ class TestGetChartData(ZulipTestCase):
for i, subgroup in enumerate(user_subgroups)])
FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
def test_number_of_humans(self):
# type: () -> None
def test_number_of_humans(self) -> None:
stat = COUNT_STATS['realm_active_humans::day']
self.insert_data(stat, [None], [])
result = self.client_get('/json/analytics/chart_data',
@@ -76,8 +71,7 @@ class TestGetChartData(ZulipTestCase):
'result': 'success',
})
def test_messages_sent_over_time(self):
# type: () -> None
def test_messages_sent_over_time(self) -> None:
stat = COUNT_STATS['messages_sent:is_bot:hour']
self.insert_data(stat, ['true', 'false'], ['false'])
result = self.client_get('/json/analytics/chart_data',
@@ -94,8 +88,7 @@ class TestGetChartData(ZulipTestCase):
'result': 'success',
})
def test_messages_sent_by_message_type(self):
# type: () -> None
def test_messages_sent_by_message_type(self) -> None:
stat = COUNT_STATS['messages_sent:message_type:day']
self.insert_data(stat, ['public_stream', 'private_message'],
['public_stream', 'private_stream'])
@@ -115,8 +108,7 @@ class TestGetChartData(ZulipTestCase):
'result': 'success',
})
def test_messages_sent_by_client(self):
# type: () -> None
def test_messages_sent_by_client(self) -> None:
stat = COUNT_STATS['messages_sent:client:day']
client1 = Client.objects.create(name='client 1')
client2 = Client.objects.create(name='client 2')
@@ -139,8 +131,7 @@ class TestGetChartData(ZulipTestCase):
'result': 'success',
})
def test_include_empty_subgroups(self):
# type: () -> None
def test_include_empty_subgroups(self) -> None:
FillState.objects.create(
property='realm_active_humans::day', end_time=self.end_times_day[0], state=FillState.DONE)
result = self.client_get('/json/analytics/chart_data',
@@ -179,8 +170,7 @@ class TestGetChartData(ZulipTestCase):
self.assertEqual(data['realm'], {})
self.assertEqual(data['user'], {})
def test_start_and_end(self):
# type: () -> None
def test_start_and_end(self) -> None:
stat = COUNT_STATS['realm_active_humans::day']
self.insert_data(stat, [None], [])
end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
@@ -202,8 +192,7 @@ class TestGetChartData(ZulipTestCase):
'end': end_time_timestamps[1]})
self.assert_json_error_contains(result, 'Start time is later than')
def test_min_length(self):
# type: () -> None
def test_min_length(self) -> None:
stat = COUNT_STATS['realm_active_humans::day']
self.insert_data(stat, [None], [])
# test min_length is too short to change anything
@@ -224,14 +213,12 @@ class TestGetChartData(ZulipTestCase):
self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
self.assertEqual(data['realm'], {'human': [0]+self.data(100)})
def test_non_existent_chart(self):
# type: () -> None
def test_non_existent_chart(self) -> None:
result = self.client_get('/json/analytics/chart_data',
{'chart_name': 'does_not_exist'})
self.assert_json_error_contains(result, 'Unknown chart name')
def test_analytics_not_running(self):
# type: () -> None
def test_analytics_not_running(self) -> None:
# try to get data for a valid chart, but before we've put anything in the database
# (e.g. before update_analytics_counts has been run)
with mock.patch('logging.warning'):
@@ -242,8 +229,7 @@ class TestGetChartData(ZulipTestCase):
class TestGetChartDataHelpers(ZulipTestCase):
# last_successful_fill is in analytics/models.py, but get_chart_data is
# the only function that uses it at the moment
def test_last_successful_fill(self):
# type: () -> None
def test_last_successful_fill(self) -> None:
self.assertIsNone(last_successful_fill('non-existant'))
a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
@@ -254,21 +240,18 @@ class TestGetChartDataHelpers(ZulipTestCase):
fillstate.save()
self.assertEqual(last_successful_fill('property'), one_hour_before)
def test_sort_by_totals(self):
# type: () -> None
def test_sort_by_totals(self) -> None:
empty = [] # type: List[int]
value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
def test_sort_client_labels(self):
# type: () -> None
def test_sort_client_labels(self) -> None:
data = {'realm': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
class TestTimeRange(ZulipTestCase):
def test_time_range(self):
# type: () -> None
def test_time_range(self) -> None:
HOUR = timedelta(hours=1)
DAY = timedelta(days=1)
@@ -294,8 +277,7 @@ class TestTimeRange(ZulipTestCase):
[floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
class TestMapArrays(ZulipTestCase):
def test_map_arrays(self):
# type: () -> None
def test_map_arrays(self) -> None:
a = {'desktop app 1.0': [1, 2, 3],
'desktop app 2.0': [10, 12, 13],
'desktop app 3.0': [21, 22, 23],

124
analytics/views.py
View File

@@ -37,8 +37,7 @@ from zerver.models import Client, Realm, \
UserActivity, UserActivityInterval, UserProfile
@zulip_login_required
def stats(request):
# type: (HttpRequest) -> HttpResponse
def stats(request: HttpRequest) -> HttpResponse:
return render(request,
'analytics/stats.html',
context=dict(realm_name = request.user.realm.name))
@@ -117,8 +116,7 @@ def get_chart_data(request, user_profile, chart_name=REQ(),
data['display_order'] = None
return json_success(data=data)
def sort_by_totals(value_arrays):
# type: (Dict[str, List[int]]) -> List[str]
def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
totals = [(sum(values), label) for label, values in value_arrays.items()]
totals.sort(reverse=True)
return [label for total, label in totals]
@@ -129,8 +127,7 @@ def sort_by_totals(value_arrays):
# understanding the realm's traffic and the user's traffic. This function
# tries to rank the clients so that taking the first N elements of the
# sorted list has a reasonable chance of doing so.
def sort_client_labels(data):
# type: (Dict[str, Dict[str, List[int]]]) -> List[str]
def sort_client_labels(data: Dict[str, Dict[str, List[int]]]) -> List[str]:
realm_order = sort_by_totals(data['realm'])
user_order = sort_by_totals(data['user'])
label_sort_values = {} # type: Dict[str, float]
@@ -141,8 +138,7 @@ def sort_client_labels(data):
return [label for label, sort_value in sorted(label_sort_values.items(),
key=lambda x: x[1])]
def table_filtered_to_id(table, key_id):
# type: (Type[BaseCount], int) -> QuerySet
def table_filtered_to_id(table: Type[BaseCount], key_id: int) -> QuerySet:
if table == RealmCount:
return RealmCount.objects.filter(realm_id=key_id)
elif table == UserCount:
@@ -154,8 +150,7 @@ def table_filtered_to_id(table, key_id):
else:
raise AssertionError("Unknown table: %s" % (table,))
def client_label_map(name):
# type: (str) -> str
def client_label_map(name: str) -> str:
if name == "website":
return "Website"
if name.startswith("desktop app"):
@@ -174,8 +169,7 @@ def client_label_map(name):
return name[len("Zulip"):-len("Webhook")] + " webhook"
return name
def rewrite_client_arrays(value_arrays):
# type: (Dict[str, List[int]]) -> Dict[str, List[int]]
def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[int]]:
mapped_arrays = {} # type: Dict[str, List[int]]
for label, array in value_arrays.items():
mapped_label = client_label_map(label)
@@ -186,8 +180,12 @@ def rewrite_client_arrays(value_arrays):
mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
return mapped_arrays
def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroup_to_label, include_empty_subgroups):
# type: (CountStat, Type[BaseCount], int, List[datetime], Dict[Optional[str], str], bool) -> Dict[str, List[int]]
def get_time_series_by_subgroup(stat: CountStat,
table: Type[BaseCount],
key_id: int,
end_times: List[datetime],
subgroup_to_label: Dict[Optional[str], str],
include_empty_subgroups: bool) -> Dict[str, List[int]]:
queryset = table_filtered_to_id(table, key_id).filter(property=stat.property) \
.values_list('subgroup', 'end_time', 'value')
value_dicts = defaultdict(lambda: defaultdict(int)) # type: Dict[Optional[str], Dict[datetime, int]]
@@ -208,12 +206,10 @@ def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroup_to_labe
eastern_tz = pytz.timezone('US/Eastern')
def make_table(title, cols, rows, has_row_class=False):
# type: (str, List[str], List[Any], bool) -> str
def make_table(title: str, cols: List[str], rows: List[Any], has_row_class: bool=False) -> str:
if not has_row_class:
def fix_row(row):
# type: (Any) -> Dict[str, Any]
def fix_row(row: Any) -> Dict[str, Any]:
return dict(cells=row, row_class=None)
rows = list(map(fix_row, rows))
@@ -226,8 +222,7 @@ def make_table(title, cols, rows, has_row_class=False):
return content
def dictfetchall(cursor):
# type: (connection.cursor) -> List[Dict[str, Any]]
def dictfetchall(cursor: connection.cursor) -> List[Dict[str, Any]]:
"Returns all rows from a cursor as a dict"
desc = cursor.description
return [
@@ -236,8 +231,7 @@ def dictfetchall(cursor):
]
def get_realm_day_counts():
# type: () -> Dict[str, Dict[str, str]]
def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
query = '''
select
r.string_id,
@@ -275,8 +269,7 @@ def get_realm_day_counts():
min_cnt = min(raw_cnts)
max_cnt = max(raw_cnts)
def format_count(cnt):
# type: (int) -> str
def format_count(cnt: int) -> str:
if cnt == min_cnt:
good_bad = 'bad'
elif cnt == max_cnt:
@@ -291,8 +284,7 @@ def get_realm_day_counts():
return result
def realm_summary_table(realm_minutes):
# type: (Dict[str, float]) -> str
def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
query = '''
SELECT
realm.string_id,
@@ -425,8 +417,7 @@ def realm_summary_table(realm_minutes):
row['string_id'] = realm_activity_link(row['string_id'])
# Count active sites
def meets_goal(row):
# type: (Dict[str, int]) -> bool
def meets_goal(row: Dict[str, int]) -> bool:
return row['active_user_count'] >= 5
num_active_sites = len(list(filter(meets_goal, rows)))
@@ -458,8 +449,7 @@ def realm_summary_table(realm_minutes):
return content
def user_activity_intervals():
# type: () -> Tuple[mark_safe, Dict[str, float]]
def user_activity_intervals() -> Tuple[mark_safe, Dict[str, float]]:
day_end = timestamp_to_datetime(time.time())
day_start = day_end - timedelta(hours=24)
@@ -509,8 +499,7 @@ def user_activity_intervals():
content = mark_safe('<pre>' + output + '</pre>')
return content, realm_minutes
def sent_messages_report(realm):
# type: (str) -> str
def sent_messages_report(realm: str) -> str:
title = 'Recently sent messages for ' + realm
cols = [
@@ -577,18 +566,16 @@ def sent_messages_report(realm):
return make_table(title, cols, rows)
def ad_hoc_queries():
# type: () -> List[Dict[str, str]]
def get_page(query, cols, title):
# type: (str, List[str], str) -> Dict[str, str]
def ad_hoc_queries() -> List[Dict[str, str]]:
def get_page(query: str, cols: List[str], title: str) -> Dict[str, str]:
cursor = connection.cursor()
cursor.execute(query)
rows = cursor.fetchall()
rows = list(map(list, rows))
cursor.close()
def fix_rows(i, fixup_func):
# type: (int, Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None
def fix_rows(i: int,
fixup_func: Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None:
for row in rows:
row[i] = fixup_func(row[i])
@@ -750,8 +737,7 @@ def ad_hoc_queries():
@require_server_admin
@has_request_variables
def get_activity(request):
# type: (HttpRequest) -> HttpResponse
def get_activity(request: HttpRequest) -> HttpResponse:
duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
counts_content = realm_summary_table(realm_minutes) # type: str
data = [
@@ -769,8 +755,7 @@ def get_activity(request):
context=dict(data=data, title=title, is_home=True),
)
def get_user_activity_records_for_realm(realm, is_bot):
# type: (str, bool) -> QuerySet
def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet:
fields = [
'user_profile__full_name',
'user_profile__email',
@@ -789,8 +774,7 @@ def get_user_activity_records_for_realm(realm, is_bot):
records = records.select_related('user_profile', 'client').only(*fields)
return records
def get_user_activity_records_for_email(email):
# type: (str) -> List[QuerySet]
def get_user_activity_records_for_email(email: str) -> List[QuerySet]:
fields = [
'user_profile__full_name',
'query',
@@ -806,8 +790,7 @@ def get_user_activity_records_for_email(email):
records = records.select_related('user_profile', 'client').only(*fields)
return records
def raw_user_activity_table(records):
# type: (List[QuerySet]) -> str
def raw_user_activity_table(records: List[QuerySet]) -> str:
cols = [
'query',
'client',
@@ -815,8 +798,7 @@ def raw_user_activity_table(records):
'last_visit'
]
def row(record):
# type: (QuerySet) -> List[Any]
def row(record: QuerySet) -> List[Any]:
return [
record.query,
record.client.name,
@@ -828,8 +810,7 @@ def raw_user_activity_table(records):
title = 'Raw Data'
return make_table(title, cols, rows)
def get_user_activity_summary(records):
# type: (List[QuerySet]) -> Dict[str, Dict[str, Any]]
def get_user_activity_summary(records: List[QuerySet]) -> Dict[str, Dict[str, Any]]:
#: `Any` used above should be `Union(int, datetime)`.
#: However current version of `Union` does not work inside other function.
#: We could use something like:
@@ -837,8 +818,7 @@ def get_user_activity_summary(records):
#: but that would require this long `Union` to carry on throughout inner functions.
summary = {} # type: Dict[str, Dict[str, Any]]
def update(action, record):
# type: (str, QuerySet) -> None
def update(action: str, record: QuerySet) -> None:
if action not in summary:
summary[action] = dict(
count=record.count,
@@ -879,29 +859,25 @@ def get_user_activity_summary(records):
return summary
def format_date_for_activity_reports(date):
# type: (Optional[datetime]) -> str
def format_date_for_activity_reports(date: Optional[datetime]) -> str:
if date:
return date.astimezone(eastern_tz).strftime('%Y-%m-%d %H:%M')
else:
return ''
def user_activity_link(email):
# type: (str) -> mark_safe
def user_activity_link(email: str) -> mark_safe:
url_name = 'analytics.views.get_user_activity'
url = urlresolvers.reverse(url_name, kwargs=dict(email=email))
email_link = '<a href="%s">%s</a>' % (url, email)
return mark_safe(email_link)
def realm_activity_link(realm_str):
# type: (str) -> mark_safe
def realm_activity_link(realm_str: str) -> mark_safe:
url_name = 'analytics.views.get_realm_activity'
url = urlresolvers.reverse(url_name, kwargs=dict(realm_str=realm_str))
realm_link = '<a href="%s">%s</a>' % (url, realm_str)
return mark_safe(realm_link)
def realm_client_table(user_summaries):
# type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
def realm_client_table(user_summaries: Dict[str, Dict[str, Dict[str, Any]]]) -> str:
exclude_keys = [
'internal',
'name',
@@ -945,8 +921,7 @@ def realm_client_table(user_summaries):
return make_table(title, cols, rows)
def user_activity_summary_table(user_summary):
# type: (Dict[str, Dict[str, Any]]) -> str
def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
rows = []
for k, v in user_summary.items():
if k == 'name':
@@ -972,33 +947,29 @@ def user_activity_summary_table(user_summary):
title = 'User Activity'
return make_table(title, cols, rows)
def realm_user_summary_table(all_records, admin_emails):
# type: (List[QuerySet], Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]
def realm_user_summary_table(all_records: List[QuerySet],
admin_emails: Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]:
user_records = {}
def by_email(record):
# type: (QuerySet) -> str
def by_email(record: QuerySet) -> str:
return record.user_profile.email
for email, records in itertools.groupby(all_records, by_email):
user_records[email] = get_user_activity_summary(list(records))
def get_last_visit(user_summary, k):
# type: (Dict[str, Dict[str, datetime]], str) -> Optional[datetime]
def get_last_visit(user_summary: Dict[str, Dict[str, datetime]], k: str) -> Optional[datetime]:
if k in user_summary:
return user_summary[k]['last_visit']
else:
return None
def get_count(user_summary, k):
# type: (Dict[str, Dict[str, str]], str) -> str
def get_count(user_summary: Dict[str, Dict[str, str]], k: str) -> str:
if k in user_summary:
return user_summary[k]['count']
else:
return ''
def is_recent(val):
# type: (Optional[datetime]) -> bool
def is_recent(val: Optional[datetime]) -> bool:
age = timezone_now() - val
return age.total_seconds() < 5 * 60
@@ -1020,8 +991,7 @@ def realm_user_summary_table(all_records, admin_emails):
row = dict(cells=cells, row_class=row_class)
rows.append(row)
def by_used_time(row):
# type: (Dict[str, Any]) -> str
def by_used_time(row: Dict[str, Any]) -> str:
return row['cells'][3]
rows = sorted(rows, key=by_used_time, reverse=True)
@@ -1044,8 +1014,7 @@ def realm_user_summary_table(all_records, admin_emails):
return user_records, content
@require_server_admin
def get_realm_activity(request, realm_str):
# type: (HttpRequest, str) -> HttpResponse
def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
data = [] # type: List[Tuple[str, str]]
all_user_records = {} # type: Dict[str, Any]
@@ -1080,8 +1049,7 @@ def get_realm_activity(request, realm_str):
)
@require_server_admin
def get_user_activity(request, email):
# type: (HttpRequest, str) -> HttpResponse
def get_user_activity(request: HttpRequest, email: str) -> HttpResponse:
records = get_user_activity_records_for_email(email)
data = [] # type: List[Tuple[str, str]]