Release Zulip Server 1.6.0.

This was needed before Zulip switched to using a pinned node version.

.editorconfig

@@ -6,7 +6,7 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
-[*.{sh,py,js, json,yml,xml, css, md,markdown, handlebars,html}]
+[*.{sh,py,js,json,yml,xml,css,md,markdown,handlebars,html}]
 indent_style = space
 indent_size = 4
 

.eslintignore

@@ -1,3 +1,3 @@
-static/js/bundle.js
 static/js/blueslip.js
 puppet/zulip_ops/files/statsd/local.js
+static/webpack-bundles

.eslintrc.json

@@ -14,25 +14,56 @@
         "Dropbox": false,
         "SockJS": false,
         "marked": false,
+        "moment": false,
         "i18n": false,
+        "DynamicText": false,
         "bridge": false,
         "page_params": false,
         "status_classes": false,
         "password_quality": false,
+        "attachments_ui": false,
         "csrf_token": false,
         "typeahead_helper": false,
+        "pygments_data": false,
         "popovers": false,
         "server_events": false,
+        "server_events_dispatch": false,
         "ui": false,
+        "ui_report": false,
+        "ui_util": false,
+        "lightbox": false,
         "stream_color": false,
         "people": false,
         "navigate": false,
+        "settings_account": false,
+        "settings_display": false,
+        "settings_notifications": false,
+        "settings_muting": false,
+        "settings_lab": false,
+        "settings_bots": false,
+        "settings_sections": false,
+        "settings_emoji": false,
+        "settings_org": false,
+        "settings_users": false,
+        "settings_streams": false,
+        "settings_filters": false,
         "settings": false,
         "resize": false,
         "loading": false,
+        "typing": false,
+        "typing_events": false,
+        "typing_data": false,
+        "typing_status": false,
         "compose": false,
+        "compose_actions": false,
+        "compose_state": false,
         "compose_fade": false,
+        "overlays": false,
+        "stream_create": false,
+        "stream_edit": false,
         "subs": false,
+        "stream_muting": false,
+        "stream_events": false,
         "timerender": false,
         "message_live_update": false,
         "message_edit": false,
@@ -40,8 +71,10 @@
         "composebox_typeahead": false,
         "search": false,
         "topic_list": false,
+        "topic_generator": false,
         "gear_menu": false,
         "hashchange": false,
+        "hash_util": false,
         "message_list": false,
         "Filter": false,
         "pointer": false,
@@ -54,28 +87,41 @@
         "Socket": false,
         "channel": false,
         "components": false,
-        "viewport": false,
+        "message_viewport": false,
+        "upload_widget": false,
         "avatar": false,
+        "realm_icon": false,
         "feature_flags": false,
         "search_suggestion": false,
         "referral": false,
         "notifications": false,
         "message_flags": false,
         "bot_data": false,
+        "stream_sort": false,
         "stream_list": false,
+        "stream_popover": false,
+        "narrow_state": false,
         "narrow": false,
+        "narrow_state": false,
+        "admin_sections": false,
         "admin": false,
         "stream_data": false,
+        "list_util": false,
         "muting": false,
         "Dict": false,
         "unread": false,
         "alert_words_ui": false,
         "message_store": false,
+        "message_util": false,
+        "message_events": false,
+        "message_fetch": false,
         "favicon": false,
         "condense": false,
+        "list_render": false,
         "floating_recipient_bar": false,
         "tab_bar": false,
         "emoji": false,
+        "presence": false,
         "activity": false,
         "invite": false,
         "colorspace": false,
@@ -84,15 +130,26 @@
         "templates": false,
         "alert_words": false,
         "fenced_code": false,
+        "markdown": false,
         "echo": false,
         "localstorage": false,
+        "localStorage": false,
         "current_msg_list": true,
         "home_msg_list": false,
         "pm_list": false,
+        "pm_conversations": false,
+        "recent_senders": false,
         "unread_ui": false,
+        "unread_ops": false,
         "user_events": false,
         "Plotly": false,
-        "emoji_codes": false
+        "emoji_codes": false,
+        "drafts": false,
+        "katex": false,
+        "Clipboard": false,
+        "emoji_picker": false,
+        "hotspots": false,
+        "compose_ui": false
     },
     "rules": {
         "no-restricted-syntax": 0,
@@ -135,7 +192,6 @@
         "no-new-func": "error",
         "space-before-function-paren": ["error", { "anonymous": "always", "named": "never", "asyncArrow": "always" }],
         "no-param-reassign": 0,
-        "prefer-spread": "error",
         "arrow-spacing": ["error", { "before": true, "after": true }],
         "no-alert": 2,
         "no-array-constructor": 2,

.gitignore

@@ -18,8 +18,11 @@ coverage/
 /zproject/dev-secrets.conf
 static/js/bundle.js
 static/generated/emoji
+static/generated/pygments_data.js
 static/generated/github-contributors.json
 static/locale/language_options.json
+static/third/emoji-data
+static/webpack-bundles
 /node_modules
 /staticfiles.json
 npm-debug.log
@@ -27,3 +30,5 @@ npm-debug.log
 var/*
 .vscode/
 tools/conf.ini
+tools/custom_provision
+api/bots/john/assets/var/database.db

.gitlint

@@ -0,0 +1,13 @@
+[general]
+ignore=title-trailing-punctuation, body-min-length, body-is-missing
+
+extra-path=tools/lib/gitlint-rules.py
+
+[title-match-regex]
+regex=^.+\.$
+
+[title-max-length]
+line-length=76
+
+[body-max-line-length]
+line-length=76

.travis.yml

@@ -1,17 +1,31 @@
+# See https://zulip.readthedocs.io/en/latest/events-system.html for
+# high-level documentation on our Travis CI setup.
 dist: trusty
-before_install:
-   - nvm install 0.10
 install:
   # Disable Travis CI's built-in NVM installation
   - mv ~/.nvm ~/.travis-nvm-disabled
+
+  # Install coveralls, the library for the code coverage reporting tool we use
   - pip install coveralls
+
+  # This is the main setup job for the test suite
   - tools/travis/setup-$TEST_SUITE
+
+  # Clean any virtualenvs that are not in use to avoid our cache
+  # becoming huge.  TODO: Add similar cleanup code for the other caches.
   - tools/clean-venv-cache --travis
+script:
+  # We unset GEM_PATH here as a hack to work around Travis CI having
+  # broken running their system puppet with Ruby.  See
+  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
 cache:
   - apt: false
   - directories:
     - $HOME/zulip-venv-cache
-    - node_modules
+    - $HOME/zulip-npm-cache
+    - $HOME/zulip-emoji-cache
     - $HOME/node
 env:
   global:
@@ -20,13 +34,10 @@ env:
     - COVERALLS_SERVICE_NAME=travis-pro
     - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
     - BOTO_CONFIG=/tmp/nowhere
-  matrix:
-    - TEST_SUITE=frontend
-    - TEST_SUITE=backend
 language: python
-python:
-  - "2.7"
-  - "3.4"
+# We run all of our test suites for both Python 2.7 and 3.4, with the
+# exception of static analysis, which is just run once (and checks
+# against both Python versions).
 matrix:
   include:
     - python: "3.4"
@@ -35,20 +46,31 @@ matrix:
       env: TEST_SUITE=production
     - python: "2.7"
       env: TEST_SUITE=production
-# command to run tests
-script:
-  - unset GEM_PATH
-  - ./tools/travis/$TEST_SUITE
+    - python: "2.7"
+      env: TEST_SUITE=frontend
+    - python: "3.4"
+      env: TEST_SUITE=frontend
+    - python: "2.7"
+      env: TEST_SUITE=backend
+    - python: "3.4"
+      env: TEST_SUITE=backend
 sudo: required
 services:
 - docker
 addons:
   artifacts:
     paths:
+      # Casper debugging data (screenshots, etc.) is super useful for
+      # debugging test flakes.
       - $(ls var/casper/* | tr "\n" ":")
       - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
   postgresql: "9.3"
 after_success:
   coveralls
 notifications:
-  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
+  webhooks:
+    urls:
+      - https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
+      - https://zulip.org/zulipbot/travis
+    on_success: always
+    on_failure: always

README.md

@@ -17,7 +17,7 @@ previews, group private messages, audible notifications,
 missed-message emails, desktop apps, and much more.
 
 Further information on the Zulip project and its features can be found
-at https://www.zulip.org.
+at <https://www.zulip.org>.
 
 [![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master) [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/) [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
 
@@ -25,18 +25,15 @@ at https://www.zulip.org.
 
 There are several places online where folks discuss Zulip.
 
-One of those places is our [public Zulip instance](https://chat.zulip.org/).
-You can go through the simple signup process at that link, and then you
-will soon be talking to core Zulip developers and other users.  To get
-help in real time, you will have the best luck finding core developers
-roughly between 16:00 UTC and 23:59 UTC.  Most questions get a reply
-within minutes to a few hours, depending on time of day.
+* The primary place is the
+  [Zulip development community Zulip server](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html).
 
-For Google Summer of Code students and applicants, we have [a mailing
-list](https://groups.google.com/forum/#!forum/zulip-gsoc) for help,
-questions, and announcements.
+* For Google Summer of Code students and applicants, we have
+[a mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc)
+for help, questions, and announcements.  But it's often simpler to
+visit chat.zulip.org instead.
 
-We have
+* We have
 [a public mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
 that is currently pretty low traffic because most discussions happen
 in our public Zulip instance.  We use it to announce Zulip developer
@@ -47,13 +44,14 @@ ask for generic help getting started as a contributor (e.g. because
 you want to do Google Summer of Code).  The rest of this page covers
 how to get involved in the Zulip project in detail.
 
-Zulip also has a [blog](https://blog.zulip.org/).
+* Zulip also has a [blog](https://blog.zulip.org/) and
+  [twitter account](https://twitter.com/zuliposs).
 
-Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
-track Zulip-related issues (and store our code, of course).
+* Last but not least, we use [GitHub](https://github.com/zulip/zulip)
+to track Zulip-related issues (and store our code, of course).
 Anybody with a GitHub account should be able to create Issues there
-pertaining to bugs or enhancement requests.  We also use Pull
-Requests as our primary mechanism to receive code contributions.
+pertaining to bugs or enhancement requests.  We also use Pull Requests
+as our primary mechanism to receive code contributions.
 
 The Zulip community has a [Code of Conduct][code-of-conduct].
 
@@ -68,17 +66,21 @@ installation guide][dev-install].
 Zulip in production supports Ubuntu 14.04 Trusty and Ubuntu 16.04
 Xenial. Work is ongoing on adding support for additional
 platforms. The installation process is documented at
-https://zulip.org/server.html and in more detail in [the
+<https://zulip.org/server.html> and in more detail in [the
 documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
 
 ## Ways to contribute
 
-Zulip welcomes all forms of contributions!  The page documents the
+Zulip welcomes all forms of contributions!  This page documents the
 Zulip development process.
 
 * **Pull requests**. Before a pull request can be merged, you need to
 sign the [Dropbox Contributor License Agreement][cla].  Also,
 please skim our [commit message style guidelines][doc-commit-style].
+We encourage early pull requests for work in progress. Prefix the title
+of your pull request with `[WIP]` and reference it when asking for
+community feedback. When you are ready for final review, remove
+the `[WIP]`.
 
 * **Testing**. The Zulip automated tests all run automatically when
 you submit a pull request, but you can also run them all in your
@@ -102,10 +104,10 @@ relevant list!  Please report any security issues you discover to
 zulip-security@googlegroups.com.
 
 * **App codebases**. This repository is for the Zulip server and web
-app (including most integrations); the [desktop][], [Android][], and
-[iOS][] apps, are separate repositories, as are our [experimental
-React Native iOS app][ios-exp] and [alpha Electron desktop
-app][electron].
+app (including most integrations); the
+[React Native Mobile iOS app][ios-exp], [Android app][Android],
+[new Electron desktop app][electron], and
+[legacy QT-based desktop app][desktop] are all separate repositories.
 
 * **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
 integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
@@ -118,6 +120,14 @@ and [Trello][]), plus [node.js API bindings][node], an [isomorphic
 [translating documentation][transifex] if you're interested in
 contributing!
 
+* **Code Reviews**. Zulip is all about community and helping each
+other out.  Check out [#code review][code-review] on
+[chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html)
+to help review PRs and give comments on other people's work. Everyone is
+welcome to participate, even those new to Zulip! Even just checking out
+the code, manually testing it, and posting on whether or not it worked
+is valuable.
+
 [cla]: https://opensource.dropbox.com/cla/
 [code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
 [dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
@@ -130,7 +140,6 @@ contributing!
 [gh-issues]: https://github.com/zulip/zulip/issues
 [desktop]: https://github.com/zulip/zulip-desktop
 [android]: https://github.com/zulip/zulip-android
-[ios]: https://github.com/zulip/zulip-ios
 [ios-exp]: https://github.com/zulip/zulip-mobile
 [email-android]: https://groups.google.com/forum/#!forum/zulip-android
 [email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
@@ -145,16 +154,27 @@ contributing!
 [tsearch]: https://github.com/zulip/tsearch_extras
 [transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
 [z-org]: https://github.com/zulip/zulip.github.io
+[code-review]: https://chat.zulip.org/#narrow/stream/code.20review
 
 ## Google Summer of Code
 
 We participated in
-[GSoC](https://developers.google.com/open-source/gsoc/) last year and
-hope to do so again in 2017.  For guidance, please read
+[GSoC](https://developers.google.com/open-source/gsoc/) in 2016 (with
+[great results](https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/))
+and are participating in 2017 as well.  For guidance, please read
 [our GSoC instructions and ideas page](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
 and feel free to email
 [our GSoC mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc).
 
+**Note**: For GSoC 2017, we will be focused on making our
+[React Native app](https://github.com/zulip/zulip-mobile) better
+rather than developing the
+[Java Android app](https://github.com/zulip/zulip-android) and
+[React Native app](https://github.com/zulip/zulip-mobile) in
+parallel. You can review
+[our detailed plan](https://github.com/zulip/zulip-android/blob/master/android-strategy.md)
+for further details on the motivation and logistics.
+
 ## How to get involved with contributing to Zulip
 
 First, subscribe to the Zulip [development discussion mailing
@@ -202,16 +222,25 @@ Another way to find issues in Zulip is to take advantage of our
 our issues into areas like admin, compose, emoji, hotkeys, i18n,
 onboarding, search, etc.  You can see this here:
 
-[https://github.com/zulip/zulip/labels]
+<https://github.com/zulip/zulip/labels>
 
 Click on any of the "area:" labels and you will see all the tickets
 related to your area of interest.
 
-If you're excited about helping with an open issue, just post on the
-conversation thread that you're working on it.  You're encouraged to
-ask questions on how to best implement or debug your changes -- the
-Zulip maintainers are excited to answer questions to help you stay
-unblocked and working efficiently.
+If you're excited about helping with an open issue, make sure to claim
+the issue by commenting the following in the comment section:
+"**@zulipbot** claim". **@zulipbot** will assign you to the issue and
+label the issue as **in progress**. For more details, check out
+[**@zulipbot**](https://github.com/zulip/zulipbot).
+
+You're encouraged to ask questions on how to best implement or debug
+your changes -- the Zulip maintainers are excited to answer questions
+to help you stay unblocked and working efficiently. It's great to ask
+questions in comments on GitHub issues and pull requests, or [on
+chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html). We'll
+direct longer discussions to Zulip chat, but please post a summary of
+what you learned from the chat, or link to the conversation, in a
+comment on the GitHub issue.
 
 We also welcome suggestions of features that you feel would be
 valuable or changes that you feel would make Zulip a better open
@@ -241,9 +270,16 @@ Feedback on how to make this development process more efficient, fun,
 and friendly to new contributors is very welcome!  Just send an email
 to the Zulip Developers list with your thoughts.
 
+When you feel like you have completed your work on an issue, post your
+PR to the
+[#code review](https://chat.zulip.org/#narrow/stream/code.20review)
+stream on [chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html).
+This is our lightweight process that gives other developers the
+opportunity to give you comments and suggestions on your work.
+
 ## License
 
-Copyright 2011-2016 Dropbox, Inc. and contributors
+Copyright 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

Vagrantfile

@@ -99,8 +99,21 @@ set -o pipefail
 if [ -d "/sys/fs/selinux" ]; then
   sudo mount -o remount,ro /sys/fs/selinux
 fi
+
+# Set default locale, this prevents errors if the user has another locale set.
+if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
+    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
+fi
+
+# Provision the development environment
 ln -nsf /srv/zulip ~/zulip
 /srv/zulip/tools/provision
+
+# Run any custom provision hooks the user has configured
+if [ -f /srv/zulip/tools/custom_provision ]; then
+  chmod +x /srv/zulip/tools/custom_provision
+  /srv/zulip/tools/custom_provision
+fi
 SCRIPT
 
   config.vm.provision "shell",

analytics/lib/counts.py

@@ -1,19 +1,24 @@
-from django.db import connection, models
-from django.utils import timezone
 from django.conf import settings
-from datetime import timedelta, datetime
+from django.db import connection, models
+from django.db.models import F
 
 from analytics.models import InstallationCount, RealmCount, \
-    UserCount, StreamCount, BaseCount, FillState, installation_epoch
-from zerver.models import Realm, UserProfile, Message, Stream, models
-from zerver.lib.timestamp import floor_to_day
+    UserCount, StreamCount, BaseCount, FillState, Anomaly, installation_epoch, \
+    last_successful_fill
+from zerver.models import Realm, UserProfile, Message, Stream, \
+    UserActivityInterval, RealmAuditLog, models
+from zerver.lib.timestamp import floor_to_day, floor_to_hour, ceiling_to_day, \
+    ceiling_to_hour
 
-from typing import Any, Optional, Type, Tuple, Text
+from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, Union
 
+from collections import defaultdict, OrderedDict
+from datetime import timedelta, datetime
 import logging
 import time
 
 ## Logging setup ##
+
 log_format = '%(asctime)s %(levelname)-8s %(message)s'
 logging.basicConfig(format=log_format)
 
@@ -25,47 +30,68 @@ logger = logging.getLogger("zulip.management")
 logger.setLevel(logging.INFO)
 logger.addHandler(file_handler)
 
-# First post office in Boston
-MIN_TIME = datetime(1639, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+# You can't subtract timedelta.max from a datetime, so use this instead
+TIMEDELTA_MAX = timedelta(days=365*1000)
+
+## Class definitions ##
 
 class CountStat(object):
     HOUR = 'hour'
     DAY = 'day'
     FREQUENCIES = frozenset([HOUR, DAY])
-    # Allowed intervals are HOUR, DAY, and, GAUGE
-    GAUGE = 'gauge'
 
-    def __init__(self, property, zerver_count_query, filter_args, group_by, frequency, is_gauge):
-        # type: (str, ZerverCountQuery, Dict[str, bool], Optional[Tuple[models.Model, str]], str, bool) -> None
+    def __init__(self, property, data_collector, frequency, interval=None):
+        # type: (str, DataCollector, str, Optional[timedelta]) -> None
         self.property = property
-        self.zerver_count_query = zerver_count_query
+        self.data_collector = data_collector
         # might have to do something different for bitfields
-        self.filter_args = filter_args
-        self.group_by = group_by
         if frequency not in self.FREQUENCIES:
-            raise ValueError("Unknown frequency: %s" % (frequency,))
+            raise AssertionError("Unknown frequency: %s" % (frequency,))
         self.frequency = frequency
-        self.interval = self.GAUGE if is_gauge else frequency
+        if interval is not None:
+            self.interval = interval
+        elif frequency == CountStat.HOUR:
+            self.interval = timedelta(hours=1)
+        else:  # frequency == CountStat.DAY
+            self.interval = timedelta(days=1)
 
     def __unicode__(self):
         # type: () -> Text
         return u"<CountStat: %s>" % (self.property,)
 
-class ZerverCountQuery(object):
-    def __init__(self, zerver_table, analytics_table, query):
-        # type: (Type[models.Model], Type[BaseCount], Text) -> None
-        self.zerver_table = zerver_table
-        self.analytics_table = analytics_table
-        self.query = query
+class LoggingCountStat(CountStat):
+    def __init__(self, property, output_table, frequency):
+        # type: (str, Type[BaseCount], str) -> None
+        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 
-def do_update_fill_state(fill_state, end_time, state):
-    # type: (FillState, datetime, int) -> None
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
+class DependentCountStat(CountStat):
+    def __init__(self, property, data_collector, frequency, interval=None, dependencies=[]):
+        # type: (str, DataCollector, str, Optional[timedelta], List[str]) -> None
+        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
+        self.dependencies = dependencies
+
+class DataCollector(object):
+    def __init__(self, output_table, pull_function):
+        # type: (Type[BaseCount], Optional[Callable[[str, datetime, datetime], int]]) -> None
+        self.output_table = output_table
+        self.pull_function = pull_function
+
+## CountStat-level operations ##
 
 def process_count_stat(stat, fill_to_time):
     # type: (CountStat, datetime) -> None
+    if stat.frequency == CountStat.HOUR:
+        time_increment = timedelta(hours=1)
+    elif stat.frequency == CountStat.DAY:
+        time_increment = timedelta(days=1)
+    else:
+        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
+
+    if floor_to_hour(fill_to_time) != fill_to_time:
+        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
+    if fill_to_time.tzinfo is None:
+        raise ValueError("fill_to_time must be timezone aware: %s" % (fill_to_time,))
+
     fill_state = FillState.objects.filter(property=stat.property).first()
     if fill_state is None:
         currently_filled = installation_epoch()
@@ -75,81 +101,88 @@ def process_count_stat(stat, fill_to_time):
         logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
     elif fill_state.state == FillState.STARTED:
         logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
-        do_delete_count_stat_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - timedelta(hours = 1)
+        do_delete_counts_at_hour(stat, fill_state.end_time)
+        currently_filled = fill_state.end_time - time_increment
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         logger.info("UNDO DONE %s" % (stat.property,))
     elif fill_state.state == FillState.DONE:
         currently_filled = fill_state.end_time
     else:
-        raise ValueError("Unknown value for FillState.state: %s." % (fill_state.state,))
+        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
 
-    currently_filled = currently_filled + timedelta(hours = 1)
+    if isinstance(stat, DependentCountStat):
+        for dependency in stat.dependencies:
+            dependency_fill_time = last_successful_fill(dependency)
+            if dependency_fill_time is None:
+                logger.warning("DependentCountStat %s run before dependency %s." %
+                               (stat.property, dependency))
+                return
+            fill_to_time = min(fill_to_time, dependency_fill_time)
+
+    currently_filled = currently_filled + time_increment
     while currently_filled <= fill_to_time:
-        logger.info("START %s %s %s" % (stat.property, stat.interval, currently_filled))
+        logger.info("START %s %s" % (stat.property, currently_filled))
         start = time.time()
         do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
         do_fill_count_stat_at_hour(stat, currently_filled)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
-        currently_filled = currently_filled + timedelta(hours = 1)
-        logger.info("DONE %s %s (%dms)" % (stat.property, stat.interval, (end-start)*1000))
+        currently_filled = currently_filled + time_increment
+        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
 
-# We assume end_time is on an hour boundary, and is timezone aware.
-# It is the caller's responsibility to enforce this!
+def do_update_fill_state(fill_state, end_time, state):
+    # type: (FillState, datetime, int) -> None
+    fill_state.end_time = end_time
+    fill_state.state = state
+    fill_state.save()
+
+# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
+# and is timezone aware. It is the caller's responsibility to enforce this!
 def do_fill_count_stat_at_hour(stat, end_time):
     # type: (CountStat, datetime) -> None
-    if stat.frequency == CountStat.DAY and (end_time != floor_to_day(end_time)):
-        return
-
-    if stat.interval == CountStat.HOUR:
-        start_time = end_time - timedelta(hours = 1)
-    elif stat.interval == CountStat.DAY:
-        start_time = end_time - timedelta(days = 1)
-    else: # stat.interval == CountStat.GAUGE
-        start_time = MIN_TIME
-
-    do_pull_from_zerver(stat, start_time, end_time)
+    start_time = end_time - stat.interval
+    if not isinstance(stat, LoggingCountStat):
+        timer = time.time()
+        assert(stat.data_collector.pull_function is not None)
+        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
+        logger.info("%s run pull_function (%dms/%sr)" %
+                    (stat.property, (time.time()-timer)*1000, rows_added))
     do_aggregate_to_summary_table(stat, end_time)
 
-def do_delete_count_stat_at_hour(stat, end_time):
+def do_delete_counts_at_hour(stat, end_time):
     # type: (CountStat, datetime) -> None
-    UserCount.objects.filter(property = stat.property, end_time = end_time).delete()
-    StreamCount.objects.filter(property = stat.property, end_time = end_time).delete()
-    RealmCount.objects.filter(property = stat.property, end_time = end_time).delete()
-    InstallationCount.objects.filter(property = stat.property, end_time = end_time).delete()
-
-def do_drop_all_analytics_tables():
-    # type: () -> None
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
+    if isinstance(stat, LoggingCountStat):
+        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
+        if stat.data_collector.output_table in [UserCount, StreamCount]:
+            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
+    else:
+        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
+        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
+        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
+        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
 
 def do_aggregate_to_summary_table(stat, end_time):
     # type: (CountStat, datetime) -> None
     cursor = connection.cursor()
 
     # Aggregate into RealmCount
-    analytics_table = stat.zerver_count_query.analytics_table
-    if analytics_table in (UserCount, StreamCount):
+    output_table = stat.data_collector.output_table
+    if output_table in (UserCount, StreamCount):
         realmcount_query = """
             INSERT INTO analytics_realmcount
                 (realm_id, value, property, subgroup, end_time)
             SELECT
-                zerver_realm.id, COALESCE(sum(%(analytics_table)s.value), 0), '%(property)s',
-                %(analytics_table)s.subgroup, %%(end_time)s
+                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
+                %(output_table)s.subgroup, %%(end_time)s
             FROM zerver_realm
-            JOIN %(analytics_table)s
+            JOIN %(output_table)s
             ON
-            (
-                %(analytics_table)s.realm_id = zerver_realm.id AND
-                %(analytics_table)s.property = '%(property)s' AND
-                %(analytics_table)s.end_time = %%(end_time)s
-            )
-            GROUP BY zerver_realm.id, %(analytics_table)s.subgroup
-        """ % {'analytics_table': analytics_table._meta.db_table,
+                zerver_realm.id = %(output_table)s.realm_id
+            WHERE
+                %(output_table)s.property = '%(property)s' AND
+                %(output_table)s.end_time = %%(end_time)s
+            GROUP BY zerver_realm.id, %(output_table)s.subgroup
+        """ % {'output_table': output_table._meta.db_table,
                'property': stat.property}
         start = time.time()
         cursor.execute(realmcount_query, {'end_time': end_time})
@@ -164,10 +197,9 @@ def do_aggregate_to_summary_table(stat, end_time):
             sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
         FROM analytics_realmcount
         WHERE
-        (
             property = '%(property)s' AND
             end_time = %%(end_time)s
-        ) GROUP BY analytics_realmcount.subgroup
+        GROUP BY analytics_realmcount.subgroup
     """ % {'property': stat.property}
     start = time.time()
     cursor.execute(installationcount_query, {'end_time': end_time})
@@ -175,55 +207,91 @@ def do_aggregate_to_summary_table(stat, end_time):
     logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
     cursor.close()
 
-# This is the only method that hits the prod databases directly.
-def do_pull_from_zerver(stat, start_time, end_time):
-    # type: (CountStat, datetime, datetime) -> None
-    zerver_table = stat.zerver_count_query.zerver_table._meta.db_table # type: ignore
-    join_args = ' '.join('AND %s.%s = %s' % (zerver_table, key, value)
-                         for key, value in stat.filter_args.items())
-    if stat.group_by is None:
+## Utility functions called from outside counts.py ##
+
+# called from zerver/lib/actions.py; should not throw any errors
+def do_increment_logging_stat(zerver_object, stat, subgroup, event_time, increment=1):
+    # type: (Union[Realm, UserProfile, Stream], CountStat, Optional[Union[str, int, bool]], datetime, int) -> None
+    table = stat.data_collector.output_table
+    if table == RealmCount:
+        id_args = {'realm': zerver_object}
+    elif table == UserCount:
+        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
+    else:  # StreamCount
+        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
+
+    if stat.frequency == CountStat.DAY:
+        end_time = ceiling_to_day(event_time)
+    else:  # CountStat.HOUR:
+        end_time = ceiling_to_hour(event_time)
+
+    row, created = table.objects.get_or_create(
+        property=stat.property, subgroup=subgroup, end_time=end_time,
+        defaults={'value': increment}, **id_args)
+    if not created:
+        row.value = F('value') + increment
+        row.save(update_fields=['value'])
+
+def do_drop_all_analytics_tables():
+    # type: () -> None
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+    Anomaly.objects.all().delete()
+
+## DataCollector-level operations ##
+
+def do_pull_by_sql_query(property, start_time, end_time, query, group_by):
+    # type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int
+    if group_by is None:
         subgroup = 'NULL'
         group_by_clause  = ''
     else:
-        subgroup = '%s.%s' % (stat.group_by[0]._meta.db_table, stat.group_by[1])
+        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
         group_by_clause = ', ' + subgroup
 
-    # We do string replacement here because passing join_args as a param
-    # may result in problems when running cursor.execute; we do
-    # the string formatting prior so that cursor.execute runs it as sql
-    query_ = stat.zerver_count_query.query % {'zerver_table': zerver_table,
-                                              'property': stat.property,
-                                              'join_args': join_args,
-                                              'subgroup': subgroup,
-                                              'group_by_clause': group_by_clause}
+    # We do string replacement here because cursor.execute will reject a
+    # group_by_clause given as a param.
+    # We pass in the datetimes as params to cursor.execute so that we don't have to
+    # think about how to convert python datetimes to SQL datetimes.
+    query_ = query % {'property': property, 'subgroup': subgroup,
+                      'group_by_clause': group_by_clause}
     cursor = connection.cursor()
-    start = time.time()
     cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
-    end = time.time()
-    logger.info("%s do_pull_from_zerver (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
+    rowcount = cursor.rowcount
     cursor.close()
+    return rowcount
 
-count_user_by_realm_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(%(zerver_table)s),'%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realm
-    JOIN zerver_userprofile
-    ON
-    (
-        zerver_userprofile.realm_id = zerver_realm.id AND
-        zerver_userprofile.date_joined >= %%(time_start)s AND
-        zerver_userprofile.date_joined < %%(time_end)s
-        %(join_args)s
-    )
-    WHERE
-        zerver_realm.date_created < %%(time_end)s
-    GROUP BY zerver_realm.id %(group_by_clause)s
-"""
-zerver_count_user_by_realm = ZerverCountQuery(UserProfile, RealmCount, count_user_by_realm_query)
+def sql_data_collector(output_table, query, group_by):
+    # type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector
+    def pull_function(property, start_time, end_time):
+        # type: (str, datetime, datetime) -> int
+        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
+    return DataCollector(output_table, pull_function)
+
+def do_pull_minutes_active(property, start_time, end_time):
+    # type: (str, datetime, datetime) -> int
+    user_activity_intervals = UserActivityInterval.objects.filter(
+        end__gt=start_time, start__lt=end_time
+    ).select_related(
+        'user_profile'
+    ).values_list(
+        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
+
+    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
+    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
+        start = max(start_time, interval_start)
+        end = min(end_time, interval_end)
+        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
+
+    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
+                      end_time=end_time, value=int(seconds // 60))
+            for ids, seconds in seconds_active.items() if seconds >= 60]
+    UserCount.objects.bulk_create(rows)
+    return len(rows)
 
-# currently .sender_id is only Message specific thing
 count_message_by_user_query = """
     INSERT INTO analytics_usercount
         (user_id, realm_id, value, property, subgroup, end_time)
@@ -232,17 +300,162 @@ count_message_by_user_query = """
     FROM zerver_userprofile
     JOIN zerver_message
     ON
-    (
-        zerver_message.sender_id = zerver_userprofile.id AND
+        zerver_userprofile.id = zerver_message.sender_id
+    WHERE
+        zerver_userprofile.date_joined < %%(time_end)s AND
         zerver_message.pub_date >= %%(time_start)s AND
         zerver_message.pub_date < %%(time_end)s
-        %(join_args)s
-    )
-    WHERE
-            zerver_userprofile.date_joined < %%(time_end)s
     GROUP BY zerver_userprofile.id %(group_by_clause)s
 """
-zerver_count_message_by_user = ZerverCountQuery(Message, UserCount, count_message_by_user_query)
+
+# Note: ignores the group_by / group_by_clause.
+count_message_type_by_user_query = """
+    INSERT INTO analytics_usercount
+            (realm_id, user_id, value, property, subgroup, end_time)
+    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
+    FROM
+    (
+        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
+        CASE WHEN
+                  zerver_recipient.type = 1 THEN 'private_message'
+             WHEN
+                  zerver_recipient.type = 3 THEN 'huddle_message'
+             WHEN
+                  zerver_stream.invite_only = TRUE THEN 'private_stream'
+             ELSE 'public_stream'
+        END
+        message_type
+
+        FROM zerver_userprofile
+        JOIN zerver_message
+        ON
+            zerver_userprofile.id = zerver_message.sender_id AND
+            zerver_message.pub_date >= %%(time_start)s AND
+            zerver_message.pub_date < %%(time_end)s
+        JOIN zerver_recipient
+        ON
+            zerver_message.recipient_id = zerver_recipient.id
+        LEFT JOIN zerver_stream
+        ON
+            zerver_recipient.type_id = zerver_stream.id
+        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
+    ) AS subquery
+    GROUP BY realm_id, id, message_type
+"""
+
+# This query joins to the UserProfile table since all current queries that
+# use this also subgroup on UserProfile.is_bot. If in the future there is a
+# stat that counts messages by stream and doesn't need the UserProfile
+# table, consider writing a new query for efficiency.
+count_message_by_stream_query = """
+    INSERT INTO analytics_streamcount
+        (stream_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_stream
+    JOIN zerver_recipient
+    ON
+        zerver_stream.id = zerver_recipient.type_id
+    JOIN zerver_message
+    ON
+        zerver_recipient.id = zerver_message.recipient_id
+    JOIN zerver_userprofile
+    ON
+        zerver_message.sender_id = zerver_userprofile.id
+    WHERE
+        zerver_stream.date_created < %%(time_end)s AND
+        zerver_recipient.type = 2 AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s
+    GROUP BY zerver_stream.id %(group_by_clause)s
+"""
+
+# Hardcodes the query needed by active_users:is_bot:day, since that is
+# currently the only stat that uses this.
+count_user_by_realm_query = """
+    INSERT INTO analytics_realmcount
+        (realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_realm
+    JOIN zerver_userprofile
+    ON
+        zerver_realm.id = zerver_userprofile.realm_id
+    WHERE
+        zerver_realm.date_created < %%(time_end)s AND
+        zerver_userprofile.date_joined >= %%(time_start)s AND
+        zerver_userprofile.date_joined < %%(time_end)s AND
+        zerver_userprofile.is_active = TRUE
+    GROUP BY zerver_realm.id %(group_by_clause)s
+"""
+
+# Currently hardcodes the query needed for active_users_audit:is_bot:day.
+# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
+# event_type in ['user_created', 'user_deactivated', etc].
+# In particular, it's important to ensure that migrations don't cause that to happen.
+check_realmauditlog_by_user_query = """
+    INSERT INTO analytics_usercount
+        (user_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_realmauditlog ral1
+    JOIN (
+        SELECT modified_user_id, max(event_time) AS max_event_time
+        FROM zerver_realmauditlog
+        WHERE
+            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
+            event_time < %%(time_end)s
+        GROUP BY modified_user_id
+    ) ral2
+    ON
+        ral1.event_time = max_event_time AND
+        ral1.modified_user_id = ral2.modified_user_id
+    JOIN zerver_userprofile
+    ON
+        ral1.modified_user_id = zerver_userprofile.id
+    WHERE
+        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
+"""
+
+check_useractivityinterval_by_user_query = """
+    INSERT INTO analytics_usercount
+        (user_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_userprofile
+    JOIN zerver_useractivityinterval
+    ON
+        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
+    WHERE
+        zerver_useractivityinterval.end >= %%(time_start)s AND
+        zerver_useractivityinterval.start < %%(time_end)s
+    GROUP BY zerver_userprofile.id %(group_by_clause)s
+"""
+
+count_realm_active_humans_query = """
+    INSERT INTO analytics_realmcount
+        (realm_id, value, property, subgroup, end_time)
+    SELECT
+        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
+    FROM (
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = 'active_users_audit:is_bot:day' AND
+            subgroup = 'false' AND
+            end_time = %%(time_end)s
+    ) usercount1
+    JOIN (
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = '15day_actives::day' AND
+            end_time = %%(time_end)s
+    ) usercount2
+    ON
+        usercount1.user_id = usercount2.user_id
+    GROUP BY usercount1.realm_id
+"""
 
 # Currently unused and untested
 count_stream_by_realm_query = """
@@ -253,101 +466,71 @@ count_stream_by_realm_query = """
     FROM zerver_realm
     JOIN zerver_stream
     ON
-    (
-        zerver_stream.realm_id = zerver_realm.id AND
+        zerver_realm.id = zerver_stream.realm_id AND
+    WHERE
+        zerver_realm.date_created < %%(time_end)s AND
         zerver_stream.date_created >= %%(time_start)s AND
         zerver_stream.date_created < %%(time_end)s
-        %(join_args)s
-    )
-    WHERE
-        zerver_realm.date_created < %%(time_end)s
     GROUP BY zerver_realm.id %(group_by_clause)s
 """
-zerver_count_stream_by_realm = ZerverCountQuery(Stream, RealmCount, count_stream_by_realm_query)
 
-# This query violates the count_X_by_Y_query conventions in several ways. One,
-# the X table is not specified by the query name; MessageType is not a zerver
-# table. Two, it ignores the subgroup column in the CountStat object; instead,
-# it uses 'message_type' from the subquery to fill in the subgroup column.
-count_message_type_by_user_query = """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type != 2 THEN 'private_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
+## CountStat declarations ##
 
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_message.sender_id = zerver_userprofile.id AND
-            zerver_message.pub_date >= %%(time_start)s AND
-            zerver_message.pub_date < %%(time_end)s
-            %(join_args)s
-        JOIN zerver_recipient
-        ON
-            zerver_recipient.id = zerver_message.recipient_id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_stream.id = zerver_recipient.type_id
-        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-zerver_count_message_type_by_user = ZerverCountQuery(Message, UserCount, count_message_type_by_user_query)
+count_stats_ = [
+    # Messages Sent stats
+    # Stats that count the number of messages sent in various ways.
+    # These are also the set of stats that read from the Message table.
 
-# Note that this query also joins to the UserProfile table, since all
-# current queries that use this also subgroup on UserProfile.is_bot. If in
-# the future there is a query that counts messages by stream and doesn't need
-# the UserProfile table, consider writing a new query for efficiency.
-count_message_by_stream_query = """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-    (
-        zerver_recipient.type = 2 AND
-        zerver_stream.id = zerver_recipient.type_id
-    )
-    JOIN zerver_message
-    ON
-    (
-        zerver_message.recipient_id = zerver_recipient.id AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s AND
-        zerver_stream.date_created < %%(time_end)s
-        %(join_args)s
-    )
-    JOIN zerver_userprofile
-    ON zerver_userprofile.id = zerver_message.sender_id
-    GROUP BY zerver_stream.id %(group_by_clause)s
-"""
-zerver_count_message_by_stream = ZerverCountQuery(Message, StreamCount, count_message_by_stream_query)
+    CountStat('messages_sent:is_bot:hour',
+              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
+              CountStat.HOUR),
+    CountStat('messages_sent:message_type:day',
+              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
+    CountStat('messages_sent:client:day',
+              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
+              CountStat.DAY),
+    CountStat('messages_in_stream:is_bot:day',
+              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
+              CountStat.DAY),
 
-COUNT_STATS = {
-    'active_users:is_bot:day': CountStat(
-        'active_users:is_bot:day', zerver_count_user_by_realm, {'is_active': True},
-        (UserProfile, 'is_bot'), CountStat.DAY, True),
-    'messages_sent:is_bot:hour': CountStat(
-        'messages_sent:is_bot:hour', zerver_count_message_by_user, {},
-        (UserProfile, 'is_bot'), CountStat.HOUR, False),
-    'messages_sent:message_type:day': CountStat(
-        'messages_sent:message_type:day', zerver_count_message_type_by_user, {},
-        None, CountStat.DAY, False),
-    'messages_sent:client:day': CountStat(
-        'messages_sent:client:day', zerver_count_message_by_user, {},
-        (Message, 'sending_client_id'), CountStat.DAY, False),
-    'messages_sent_to_stream:is_bot:hour': CountStat(
-        'messages_sent_to_stream:is_bot', zerver_count_message_by_stream, {},
-        (UserProfile, 'is_bot'), CountStat.HOUR, False)
-}
+    # Number of Users stats
+    # Stats that count the number of active users in the UserProfile.is_active sense.
+
+    # 'active_users_audit:is_bot:day' is the canonical record of which users were
+    # active on which days (in the UserProfile.is_active sense).
+    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
+    CountStat('active_users_audit:is_bot:day',
+              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
+              CountStat.DAY),
+    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
+    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
+    # sum sequence of 'active_users_log:is_bot:day', for any realm that
+    # started after the latter stat was introduced.
+    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
+    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
+    # approximation, e.g. if a user is deactivated between the end of the
+    # day and when this stat is run, they won't be counted. However, is the
+    # simplest of the three to inspect by hand.
+    CountStat('active_users:is_bot:day',
+              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
+              CountStat.DAY, interval=TIMEDELTA_MAX),
+
+    # User Activity stats
+    # Stats that measure user activity in the UserActivityInterval sense.
+
+    CountStat('15day_actives::day',
+              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
+              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
+    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
+
+    # Dependent stats
+    # Must come after their dependencies.
+
+    # Canonical account of the number of active humans in a realm on each day.
+    DependentCountStat('realm_active_humans::day',
+                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
+                       CountStat.DAY,
+                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
+]
+
+COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])

analytics/lib/fixtures.py

@@ -8,12 +8,13 @@ from analytics.lib.time_utils import time_range
 from datetime import datetime
 from math import sqrt
 from random import gauss, random, seed
+from typing import List
 
 from six.moves import range, zip
 
 def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
                               growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
-                              frequency=CountStat.DAY, is_gauge=False, random_seed=26):
+                              frequency=CountStat.DAY, partial_sum=False, random_seed=26):
     # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
@@ -31,7 +32,7 @@ def generate_time_series_data(days=100, business_hours_base=10, non_business_hou
         the variance.
     holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
     frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    is_gauge -- If True, return partial sum of the series.
+    partial_sum -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
     if frequency == CountStat.HOUR:
@@ -49,10 +50,10 @@ def generate_time_series_data(days=100, business_hours_base=10, non_business_hou
                       [24*non_business_hours_base] * 2
         holidays = [random() < holiday_rate for i in range(days)]
     else:
-        raise ValueError("Unknown frequency: %s" % (frequency,))
+        raise AssertionError("Unknown frequency: %s" % (frequency,))
     if length < 2:
-        raise ValueError("Must be generating at least 2 data points. "
-                         "Currently generating %s" % (length,))
+        raise AssertionError("Must be generating at least 2 data points. "
+                             "Currently generating %s" % (length,))
     growth_base = growth ** (1. / (length-1))
     values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
 
@@ -63,7 +64,7 @@ def generate_time_series_data(days=100, business_hours_base=10, non_business_hou
 
     values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
               for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-    if is_gauge:
+    if partial_sum:
         for i in range(1, length):
             values[i] = values[i-1] + values[i]
     return [max(v, 0) for v in values]

analytics/lib/time_utils.py

@@ -17,7 +17,7 @@ def time_range(start, end, frequency, min_length):
         end = floor_to_day(end)
         step = timedelta(days=1)
     else:
-        raise ValueError("Unknown frequency: %s" % (frequency,))
+        raise AssertionError("Unknown frequency: %s" % (frequency,))
 
     times = []
     if min_length is not None:

analytics/management/commands/active_user_stats.py

@@ -2,7 +2,8 @@ from __future__ import absolute_import
 from __future__ import print_function
 
 from django.core.management.base import BaseCommand
-from typing import Any
+from django.utils.timezone import now as timezone_now
+from typing import Any, Dict, List
 
 from zerver.models import UserPresence, UserActivity
 from zerver.lib.utils import statsd, statsd_key
@@ -18,13 +19,13 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None
         # Get list of all active users in the last 1 week
-        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+        cutoff = timezone_now() - timedelta(minutes=30, hours=168)
 
         users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
 
         # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
         hour_buckets = [0.16, 2, 12, 24, 48, 168]
-        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+        user_info = defaultdict(dict)  # type: Dict[str, Dict[float, List[str]]]
 
         for last_presence in users:
             if last_presence.status == UserPresence.IDLE:
@@ -35,7 +36,7 @@ class Command(BaseCommand):
             for bucket in hour_buckets:
                 if bucket not in user_info[last_presence.user_profile.realm.string_id]:
                     user_info[last_presence.user_profile.realm.string_id][bucket] = []
-                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                if timezone_now() - known_active < timedelta(hours=bucket):
                     user_info[last_presence.user_profile.realm.string_id][bucket].append(last_presence.user_profile.email)
 
         for realm, buckets in user_info.items():
@@ -51,7 +52,7 @@ class Command(BaseCommand):
             for bucket in hour_buckets:
                 if bucket not in user_info[activity.user_profile.realm.string_id]:
                     user_info[activity.user_profile.realm.string_id][bucket] = []
-                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                if timezone_now() - activity.last_visit < timedelta(hours=bucket):
                     user_info[activity.user_profile.realm.string_id][bucket].append(activity.user_profile.email)
         for realm, buckets in user_info.items():
             print("Realm %s" % (realm,))

analytics/management/commands/active_user_stats_by_day.py

@@ -6,7 +6,9 @@ import pytz
 
 from optparse import make_option
 from typing import Any
+
 from django.core.management.base import BaseCommand, CommandParser
+from django.utils.timezone import now as timezone_now
 from zerver.lib.statistics import activity_averages_during_day
 
 class Command(BaseCommand):
@@ -20,9 +22,9 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None
         if options["date"] is None:
-            date = datetime.datetime.now() - datetime.timedelta(days=1)
+            date = timezone_now() - datetime.timedelta(days=1)
         else:
-            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=pytz.utc)
         print("Activity data for", date)
         print(activity_averages_during_day(date))
         print("Please note that the total registered user count is a total for today")

analytics/management/commands/analyze_mit.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-from typing import Any
+from typing import Any, Dict
 
 from optparse import make_option
 from django.core.management.base import BaseCommand, CommandParser
@@ -17,7 +17,7 @@ def compute_stats(log_level):
     logger.setLevel(log_level)
 
     one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="mit",
+    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
                                        recipient__type=Recipient.STREAM,
                                        pub_date__gt=one_week_ago)
     for bot_sender_start in ["imap.", "rcmd.", "sys."]:
@@ -30,15 +30,15 @@ def compute_stats(log_level):
                            "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                            "root@mit.edu", "nagios@mit.edu",
                            "www-data|local-realm@mit.edu"])
-    user_counts = {} # type: Dict[str, Dict[str, int]]
+    user_counts = {}  # type: Dict[str, Dict[str, int]]
     for m in mit_query.select_related("sending_client", "sender"):
         email = m.sender.email
         user_counts.setdefault(email, {})
         user_counts[email].setdefault(m.sending_client.name, 0)
         user_counts[email][m.sending_client.name] += 1
 
-    total_counts = {} # type: Dict[str, int]
-    total_user_counts = {} # type: Dict[str, int]
+    total_counts = {}  # type: Dict[str, int]
+    total_user_counts = {}  # type: Dict[str, int]
     for email, counts in user_counts.items():
         total_user_counts.setdefault(email, 0)
         for client_name, count in counts.items():
@@ -47,7 +47,7 @@ def compute_stats(log_level):
             total_user_counts[email] += count
 
     logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {} # type: Dict[int, float]
+    top_percents = {}  # type: Dict[int, float]
     for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
         top_percents[size] = 0.0
     for i, email in enumerate(sorted(total_user_counts.keys(),

analytics/management/commands/client_activity.py

@@ -6,6 +6,7 @@ from typing import Any
 from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
+from django.utils.timezone import now as timezone_now
 
 from zerver.models import UserActivity, UserProfile, Realm, \
     get_realm, get_user_profile_by_email
@@ -38,7 +39,7 @@ Usage examples:
         #
         # Importantly, this does NOT tell you anything about the relative
         # volumes of requests from clients.
-        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
+        threshold = timezone_now() - datetime.timedelta(days=7)
         client_counts = user_activity_objects.filter(
             last_visit__gt=threshold).values("client__name").annotate(
             count=Count('client__name'))

analytics/management/commands/populate_analytics_db.py

@@ -3,20 +3,21 @@ from __future__ import absolute_import, print_function
 from argparse import ArgumentParser
 
 from django.core.management.base import BaseCommand
-from django.utils import timezone
+from django.utils.timezone import now as timezone_now
 
-from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount, FillState
 from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, UserProfile, Stream, Message, Client
+from zerver.models import Realm, UserProfile, Stream, Message, Client, \
+    RealmAuditLog
 
 from datetime import datetime, timedelta
 
 from six.moves import zip
-from typing import Any, List, Optional, Text, Type, Union
+from typing import Any, Dict, List, Optional, Text, Type, Union, Mapping
 
 class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
@@ -26,38 +27,40 @@ class Command(BaseCommand):
 
     def create_user(self, email, full_name, is_staff, date_joined, realm):
         # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
-        return UserProfile.objects.create(
+        user = UserProfile.objects.create(
             email=email, full_name=full_name, is_staff=is_staff,
             realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
             api_key='42', date_joined=date_joined)
+        RealmAuditLog.objects.create(
+            realm=realm, modified_user=user, event_type='user_created',
+            event_time=user.date_joined)
+        return user
 
     def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
-                              growth, autocorrelation, spikiness, holiday_rate=0):
-        # type: (CountStat, float, float, float, float, float, float) -> List[int]
+                              growth, autocorrelation, spikiness, holiday_rate=0,
+                              partial_sum=False):
+        # type: (CountStat, float, float, float, float, float, float, bool) -> List[int]
         self.random_seed += 1
         return generate_time_series_data(
             days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
             non_business_hours_base=non_business_hours_base, growth=growth,
             autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            frequency=stat.frequency, is_gauge=(stat.interval == CountStat.GAUGE),
-            random_seed=self.random_seed)
+            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
 
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None
         do_drop_all_analytics_tables()
         # I believe this also deletes any objects with this realm as a foreign key
         Realm.objects.filter(string_id='analytics').delete()
-        Client.objects.filter(name__endswith='_').delete()
 
-        installation_time = timezone.now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone.now())
+        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
+        last_end_time = floor_to_day(timezone_now())
         realm = Realm.objects.create(
-            string_id='analytics', name='Analytics', domain='analytics.ds',
-            date_created=installation_time)
+            string_id='analytics', name='Analytics', date_created=installation_time)
         shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
 
         def insert_fixture_data(stat, fixture_data, table):
-            # type: (CountStat, Dict[Optional[str], List[int]], Type[BaseCount]) -> None
+            # type: (CountStat, Mapping[Optional[str], List[int]], Type[BaseCount]) -> None
             end_times = time_range(last_end_time, last_end_time, stat.frequency,
                                    len(list(fixture_data.values())[0]))
             if table == RealmCount:
@@ -70,54 +73,66 @@ class Command(BaseCommand):
                           value=value, **id_args)
                     for end_time, value in zip(end_times, values) if value != 0])
 
-        stat = COUNT_STATS['active_users:is_bot:day']
+        stat = COUNT_STATS['realm_active_humans::day']
         realm_data = {
-            'false': self.generate_fixture_data(stat, .1, .03, 3, .5, 3),
-            'true': self.generate_fixture_data(stat, .01, 0, 1, 0, 1)
-        } # type: Dict[Optional[str], List[int]]
+            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
+        }  # type: Mapping[Optional[str], List[int]]
         insert_fixture_data(stat, realm_data, RealmCount)
+        FillState.objects.create(property=stat.property, end_time=last_end_time,
+                                 state=FillState.DONE)
 
         stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data = {'false': self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}
+        user_data = {'false': self.generate_fixture_data(
+            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
                       'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
         insert_fixture_data(stat, realm_data, RealmCount)
+        FillState.objects.create(property=stat.property, end_time=last_end_time,
+                                 state=FillState.DONE)
 
         stat = COUNT_STATS['messages_sent:message_type:day']
         user_data = {
             'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8)}
+            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
+            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
             'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
             'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4)}
+            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
+            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
         insert_fixture_data(stat, realm_data, RealmCount)
+        FillState.objects.create(property=stat.property, end_time=last_end_time,
+                                 state=FillState.DONE)
 
-        website_ = Client.objects.create(name='website_')
-        API_ = Client.objects.create(name='API_')
-        android_ = Client.objects.create(name='android_')
-        iOS_ = Client.objects.create(name='iOS_')
-        react_native_ = Client.objects.create(name='react_native_')
-        electron_ = Client.objects.create(name='electron_')
-        barnowl_ = Client.objects.create(name='barnowl_')
-        plan9_ = Client.objects.create(name='plan9_')
+        website, created = Client.objects.get_or_create(name='website')
+        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
+        android, created = Client.objects.get_or_create(name='ZulipAndroid')
+        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
+        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
+        API, created = Client.objects.get_or_create(name='API: Python')
+        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
+        unused, created = Client.objects.get_or_create(name='unused')
+        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
 
         stat = COUNT_STATS['messages_sent:client:day']
         user_data = {
-            website_.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            barnowl_.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
+            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
+            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
-            website_.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            API_.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            android_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native_.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            electron_.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            barnowl_.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            plan9_.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0, 0)}
+            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
+            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
+            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
+            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
+            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
+            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
+            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
         insert_fixture_data(stat, realm_data, RealmCount)
+        FillState.objects.create(property=stat.property, end_time=last_end_time,
+                                 state=FillState.DONE)
 
         # TODO: messages_sent_to_stream:is_bot

analytics/management/commands/realm_stats.py

@@ -2,7 +2,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from typing import Any
+from typing import Any, List
 
 from argparse import ArgumentParser
 import datetime
@@ -10,6 +10,8 @@ import pytz
 
 from django.core.management.base import BaseCommand
 from django.db.models import Count
+from django.utils.timezone import now as timezone_now
+
 from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
     Subscription, UserMessage, get_realm
 
@@ -29,7 +31,7 @@ class Command(BaseCommand):
     def active_users(self, realm):
         # type: (Realm) -> List[UserProfile]
         # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
+        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
         return [activity.user_profile for activity in (
             UserActivity.objects.filter(user_profile__realm=realm,
                                         user_profile__is_active=True,
@@ -39,17 +41,17 @@ class Command(BaseCommand):
 
     def messages_sent_by(self, user, days_ago):
         # type: (UserProfile, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
 
     def total_messages(self, realm, days_ago):
         # type: (Realm, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
     def human_messages(self, realm, days_ago):
         # type: (Realm, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
     def api_messages(self, realm, days_ago):
@@ -58,19 +60,19 @@ class Command(BaseCommand):
 
     def stream_messages(self, realm, days_ago):
         # type: (Realm, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                      recipient__type=Recipient.STREAM).count()
 
     def private_messages(self, realm, days_ago):
         # type: (Realm, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
 
     def group_private_messages(self, realm, days_ago):
         # type: (Realm, int) -> int
-        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
 

analytics/management/commands/update_analytics_counts.py

@@ -7,18 +7,20 @@ from scripts.lib.zulip_tools import ENDC, WARNING
 
 from argparse import ArgumentParser
 from datetime import timedelta
+import time
 
 from django.core.management.base import BaseCommand
-from django.utils import timezone
+from django.utils.timezone import now as timezone_now
+from django.utils.timezone import utc as timezone_utc
 from django.utils.dateparse import parse_datetime
 from django.conf import settings
 
 from analytics.models import RealmCount, UserCount
 from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
-from zerver.lib.timestamp import datetime_to_string, is_timezone_aware
+from zerver.lib.timestamp import floor_to_hour
 from zerver.models import UserProfile, Message
 
-from typing import Any
+from typing import Any, Dict
 
 class Command(BaseCommand):
     help = """Fills Analytics tables.
@@ -30,17 +32,18 @@ class Command(BaseCommand):
         parser.add_argument('--time', '-t',
                             type=str,
                             help='Update stat tables from current state to --time. Defaults to the current time.',
-                            default=datetime_to_string(timezone.now()))
+                            default=timezone_now().isoformat())
         parser.add_argument('--utc',
-                            type=bool,
+                            action='store_true',
                             help="Interpret --time in UTC.",
                             default=False)
         parser.add_argument('--stat', '-s',
                             type=str,
                             help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument('--quiet', '-q',
-                            type=str,
-                            help="Suppress output to stdout.")
+        parser.add_argument('--verbose',
+                            action='store_true',
+                            help="Print timing information to stdout.",
+                            default=False)
 
     def handle(self, *args, **options):
         # type: (*Any, **Any) -> None
@@ -58,18 +61,31 @@ class Command(BaseCommand):
     def run_update_analytics_counts(self, options):
         # type: (Dict[str, Any]) -> None
         fill_to_time = parse_datetime(options['time'])
-        if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
 
-        if not (is_timezone_aware(fill_to_time)):
+        if options['utc']:
+            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
+        if fill_to_time.tzinfo is None:
             raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
 
-        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
+        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
 
         if options['stat'] is not None:
-            process_count_stat(COUNT_STATS[options['stat']], fill_to_time)
+            stats = [COUNT_STATS[options['stat']]]
         else:
-            for stat in COUNT_STATS.values():
-                process_count_stat(stat, fill_to_time)
+            stats = list(COUNT_STATS.values())
 
+        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
+        if options['verbose']:
+            start = time.time()
+            last = start
+
+        for stat in stats:
+            process_count_stat(stat, fill_to_time)
+            if options['verbose']:
+                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
+                last = time.time()
+
+        if options['verbose']:
+            print("Finished updating analytics counts through %s in %.3fs" %
+                  (fill_to_time, time.time() - start))
         logger.info("Finished updating analytics counts through %s" % (fill_to_time,))

analytics/management/commands/user_stats.py

@@ -7,6 +7,8 @@ import pytz
 from typing import Any
 
 from django.core.management.base import BaseCommand
+from django.utils.timezone import now as timezone_now
+
 from zerver.models import UserProfile, Realm, Stream, Message, get_realm
 from six.moves import range
 
@@ -20,8 +22,8 @@ class Command(BaseCommand):
 
     def messages_sent_by(self, user, week):
         # type: (UserProfile, int) -> int
-        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
-        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
+        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
+        end = timezone_now() - datetime.timedelta(days=week*7)
         return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
 
     def handle(self, *args, **options):

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db import migrations
+
+
+def delete_messages_sent_to_stream_stat(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    UserCount = apps.get_model('analytics', 'UserCount')
+    StreamCount = apps.get_model('analytics', 'StreamCount')
+    RealmCount = apps.get_model('analytics', 'RealmCount')
+    InstallationCount = apps.get_model('analytics', 'InstallationCount')
+    FillState = apps.get_model('analytics', 'FillState')
+
+    property = 'messages_sent_to_stream:is_bot'
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0008_add_count_indexes'),
+    ]
+
+    operations = [
+        migrations.RunPython(delete_messages_sent_to_stream_stat),
+    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db import migrations
+
+def clear_message_sent_by_message_type_values(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    UserCount = apps.get_model('analytics', 'UserCount')
+    StreamCount = apps.get_model('analytics', 'StreamCount')
+    RealmCount = apps.get_model('analytics', 'RealmCount')
+    InstallationCount = apps.get_model('analytics', 'InstallationCount')
+    FillState = apps.get_model('analytics', 'FillState')
+
+    property = 'messages_sent:message_type:day'
+    UserCount.objects.filter(property=property).delete()
+    StreamCount.objects.filter(property=property).delete()
+    RealmCount.objects.filter(property=property).delete()
+    InstallationCount.objects.filter(property=property).delete()
+    FillState.objects.filter(property=property).delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
+
+    operations = [
+        migrations.RunPython(clear_message_sent_by_message_type_values),
+    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.db import migrations
+
+
+def clear_analytics_tables(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    UserCount = apps.get_model('analytics', 'UserCount')
+    StreamCount = apps.get_model('analytics', 'StreamCount')
+    RealmCount = apps.get_model('analytics', 'RealmCount')
+    InstallationCount = apps.get_model('analytics', 'InstallationCount')
+    FillState = apps.get_model('analytics', 'FillState')
+
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0010_clear_messages_sent_values'),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_analytics_tables),
+    ]

analytics/models.py

@@ -1,24 +1,23 @@
 from django.db import models
-from django.utils import timezone
 
 from zerver.models import Realm, UserProfile, Stream, Recipient
 from zerver.lib.str_utils import ModelReprMixin
-from zerver.lib.timestamp import datetime_to_UTC, floor_to_day
+from zerver.lib.timestamp import floor_to_day
 
 import datetime
 
 from typing import Optional, Tuple, Union, Dict, Any, Text
 
 class FillState(ModelReprMixin, models.Model):
-    property = models.CharField(max_length=40, unique=True) # type: Text
-    end_time = models.DateTimeField() # type: datetime.datetime
+    property = models.CharField(max_length=40, unique=True)  # type: Text
+    end_time = models.DateTimeField()  # type: datetime.datetime
 
     # Valid states are {DONE, STARTED}
     DONE = 1
     STARTED = 2
-    state = models.PositiveSmallIntegerField() # type: int
+    state = models.PositiveSmallIntegerField()  # type: int
 
-    last_modified = models.DateTimeField(auto_now=True) # type: datetime.datetime
+    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
 
     def __unicode__(self):
         # type: () -> Text
@@ -29,11 +28,20 @@ class FillState(ModelReprMixin, models.Model):
 def installation_epoch():
     # type: () -> datetime.datetime
     earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    return floor_to_day(datetime_to_UTC(earliest_realm_creation))
+    return floor_to_day(earliest_realm_creation)
+
+def last_successful_fill(property):
+    # type: (str) -> Optional[datetime.datetime]
+    fillstate = FillState.objects.filter(property=property).first()
+    if fillstate is None:
+        return None
+    if fillstate.state == FillState.DONE:
+        return fillstate.end_time
+    return fillstate.end_time - datetime.timedelta(hours=1)
 
 # would only ever make entries here by hand
 class Anomaly(ModelReprMixin, models.Model):
-    info = models.CharField(max_length=1000) # type: Text
+    info = models.CharField(max_length=1000)  # type: Text
 
     def __unicode__(self):
         # type: () -> Text
@@ -43,40 +51,20 @@ class BaseCount(ModelReprMixin, models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32) # type: Text
-    subgroup = models.CharField(max_length=16, null=True) # type: Text
-    end_time = models.DateTimeField() # type: datetime.datetime
-    value = models.BigIntegerField() # type: int
-    anomaly = models.ForeignKey(Anomaly, null=True) # type: Optional[Anomaly]
+    property = models.CharField(max_length=32)  # type: Text
+    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[Text]
+    end_time = models.DateTimeField()  # type: datetime.datetime
+    value = models.BigIntegerField()  # type: int
+    anomaly = models.ForeignKey(Anomaly, null=True)  # type: Optional[Anomaly]
 
     class Meta(object):
         abstract = True
 
-    @staticmethod
-    def extended_id():
-        # type: () -> Tuple[str, ...]
-        raise NotImplementedError
-
-    @staticmethod
-    def key_model():
-        # type: () -> models.Model
-        raise NotImplementedError
-
 class InstallationCount(BaseCount):
 
     class Meta(object):
         unique_together = ("property", "subgroup", "end_time")
 
-    @staticmethod
-    def extended_id():
-        # type: () -> Tuple[str, ...]
-        return ()
-
-    @staticmethod
-    def key_model():
-        # type: () -> models.Model
-        return None
-
     def __unicode__(self):
         # type: () -> Text
         return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
@@ -88,16 +76,6 @@ class RealmCount(BaseCount):
         unique_together = ("realm", "property", "subgroup", "end_time")
         index_together = ["property", "end_time"]
 
-    @staticmethod
-    def extended_id():
-        # type: () -> Tuple[str, ...]
-        return ('realm_id',)
-
-    @staticmethod
-    def key_model():
-        # type: () -> models.Model
-        return Realm
-
     def __unicode__(self):
         # type: () -> Text
         return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
@@ -112,16 +90,6 @@ class UserCount(BaseCount):
         # aggregating from users to realms
         index_together = ["property", "realm", "end_time"]
 
-    @staticmethod
-    def extended_id():
-        # type: () -> Tuple[str, ...]
-        return ('user_id', 'realm_id')
-
-    @staticmethod
-    def key_model():
-        # type: () -> models.Model
-        return UserProfile
-
     def __unicode__(self):
         # type: () -> Text
         return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
@@ -136,16 +104,6 @@ class StreamCount(BaseCount):
         # aggregating from streams to realms
         index_together = ["property", "realm", "end_time"]
 
-    @staticmethod
-    def extended_id():
-        # type: () -> Tuple[str, ...]
-        return ('stream_id', 'realm_id')
-
-    @staticmethod
-    def key_model():
-        # type: () -> models.Model
-        return Stream
-
     def __unicode__(self):
         # type: () -> Text
         return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)

analytics/tests/test_counts.py

@@ -1,39 +1,47 @@
 from __future__ import absolute_import
 
+from django.apps import apps
 from django.db import models
+from django.db.models import Sum
 from django.test import TestCase
-from django.utils import timezone
+from django.utils.timezone import now as timezone_now
+from django.utils.timezone import utc as timezone_utc
 
 from analytics.lib.counts import CountStat, COUNT_STATS, process_count_stat, \
-    zerver_count_user_by_realm, zerver_count_message_by_user, \
-    zerver_count_message_by_stream, zerver_count_stream_by_realm, \
-    do_fill_count_stat_at_hour, ZerverCountQuery
+    do_fill_count_stat_at_hour, do_increment_logging_stat, DataCollector, \
+    sql_data_collector, LoggingCountStat, do_aggregate_to_summary_table, \
+    do_drop_all_analytics_tables, DependentCountStat
 from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, FillState, installation_epoch
+    UserCount, StreamCount, FillState, Anomaly, installation_epoch, \
+    last_successful_fill
+from zerver.lib.actions import do_create_user, do_deactivate_user, \
+    do_activate_user, do_reactivate_user, update_user_activity_interval
+from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, UserProfile, Message, Stream, Recipient, \
-    Huddle, Client, get_user_profile_by_email, get_client
+    Huddle, Client, UserActivityInterval, RealmAuditLog, \
+    get_user_profile_by_email, get_client
 
 from datetime import datetime, timedelta
+import ujson
 
 from six.moves import range
-from typing import Any, Type, Optional, Text, Tuple, List, Union
+from typing import Any, Dict, List, Optional, Text, Tuple, Type, Union
 
 class AnalyticsTestCase(TestCase):
     MINUTE = timedelta(seconds = 60)
     HOUR = MINUTE * 60
     DAY = HOUR * 24
-    TIME_ZERO = datetime(1988, 3, 14).replace(tzinfo=timezone.utc)
+    TIME_ZERO = datetime(1988, 3, 14).replace(tzinfo=timezone_utc)
     TIME_LAST_HOUR = TIME_ZERO - HOUR
 
     def setUp(self):
         # type: () -> None
         self.default_realm = Realm.objects.create(
-            string_id='realmtest', name='Realm Test',
-            domain='test.analytics', date_created=self.TIME_ZERO - 2*self.DAY)
+            string_id='realmtest', name='Realm Test', date_created=self.TIME_ZERO - 2*self.DAY)
         # used to generate unique names in self.create_*
         self.name_counter = 100
         # used as defaults in self.assertCountEquals
-        self.current_property = None # type: Optional[str]
+        self.current_property = None  # type: Optional[str]
 
     # Lightweight creation of users, streams, and messages
     def create_user(self, **kwargs):
@@ -103,7 +111,7 @@ class AnalyticsTestCase(TestCase):
         self.assertEqual(queryset.values_list('value', flat=True)[0], value)
 
     def assertTableState(self, table, arg_keys, arg_values):
-        # type: (Type[BaseCount], List[str], List[List[Union[int, str, Realm, UserProfile, Stream]]]) -> None
+        # type: (Type[BaseCount], List[str], List[List[Union[int, str, bool, datetime, Realm, UserProfile, Stream]]]) -> None
         """Assert that the state of a *Count table is what it should be.
 
         Example usage:
@@ -128,9 +136,10 @@ class AnalyticsTestCase(TestCase):
         defaults = {
             'property': self.current_property,
             'subgroup': None,
-            'end_time': self.TIME_ZERO}
+            'end_time': self.TIME_ZERO,
+            'value': 1}
         for values in arg_values:
-            kwargs = {} # type: Dict[str, Any]
+            kwargs = {}  # type: Dict[str, Any]
             for i in range(len(values)):
                 kwargs[arg_keys[i]] = values[i]
             for key, value in defaults.items():
@@ -147,20 +156,15 @@ class AnalyticsTestCase(TestCase):
         self.assertEqual(table.objects.count(), len(arg_values))
 
 class TestProcessCountStat(AnalyticsTestCase):
-    def make_dummy_count_stat(self, current_time):
-        # type: (datetime) -> CountStat
-        dummy_query = """INSERT INTO analytics_realmcount (realm_id, property, end_time, value)
-                                VALUES (1, 'test stat', '%(end_time)s', 22)""" % {'end_time': current_time}
-        count_stat = CountStat('test stat', ZerverCountQuery(Recipient, UserCount, dummy_query),
-                               {}, None, CountStat.HOUR, False)
-        return count_stat
+    def make_dummy_count_stat(self, property):
+        # type: (str) -> CountStat
+        query = """INSERT INTO analytics_realmcount (realm_id, value, property, end_time)
+                   VALUES (%s, 1, '%s', %%%%(time_end)s)""" % (self.default_realm.id, property)
+        return CountStat(property, sql_data_collector(RealmCount, query, None), CountStat.HOUR)
 
-    def assertFillStateEquals(self, end_time, state = FillState.DONE, property = None):
-        # type: (datetime, int, Optional[Text]) -> None
-        count_stat = self.make_dummy_count_stat(end_time)
-        if property is None:
-            property = count_stat.property
-        fill_state = FillState.objects.filter(property=property).first()
+    def assertFillStateEquals(self, stat, end_time, state=FillState.DONE):
+        # type: (CountStat, datetime, int) -> None
+        fill_state = FillState.objects.filter(property=stat.property).first()
         self.assertEqual(fill_state.end_time, end_time)
         self.assertEqual(fill_state.state, state)
 
@@ -168,29 +172,131 @@ class TestProcessCountStat(AnalyticsTestCase):
         # type: () -> None
         # process new stat
         current_time = installation_epoch() + self.HOUR
-        count_stat = self.make_dummy_count_stat(current_time)
-        property = count_stat.property
-        process_count_stat(count_stat, current_time)
-        self.assertFillStateEquals(current_time)
-        self.assertEqual(InstallationCount.objects.filter(property=property).count(), 1)
+        stat = self.make_dummy_count_stat('test stat')
+        process_count_stat(stat, current_time)
+        self.assertFillStateEquals(stat, current_time)
+        self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 1)
 
         # dirty stat
-        FillState.objects.filter(property=property).update(state=FillState.STARTED)
-        process_count_stat(count_stat, current_time)
-        self.assertFillStateEquals(current_time)
-        self.assertEqual(InstallationCount.objects.filter(property=property).count(), 1)
+        FillState.objects.filter(property=stat.property).update(state=FillState.STARTED)
+        process_count_stat(stat, current_time)
+        self.assertFillStateEquals(stat, current_time)
+        self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 1)
 
         # clean stat, no update
-        process_count_stat(count_stat, current_time)
-        self.assertFillStateEquals(current_time)
-        self.assertEqual(InstallationCount.objects.filter(property=property).count(), 1)
+        process_count_stat(stat, current_time)
+        self.assertFillStateEquals(stat, current_time)
+        self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 1)
 
         # clean stat, with update
         current_time = current_time + self.HOUR
-        count_stat = self.make_dummy_count_stat(current_time)
-        process_count_stat(count_stat, current_time)
-        self.assertFillStateEquals(current_time)
-        self.assertEqual(InstallationCount.objects.filter(property=property).count(), 2)
+        stat = self.make_dummy_count_stat('test stat')
+        process_count_stat(stat, current_time)
+        self.assertFillStateEquals(stat, current_time)
+        self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 2)
+
+    def test_bad_fill_to_time(self):
+        # type: () -> None
+        stat = self.make_dummy_count_stat('test stat')
+        with self.assertRaises(ValueError):
+            process_count_stat(stat, installation_epoch() + 65*self.MINUTE)
+        with self.assertRaises(ValueError):
+            process_count_stat(stat, installation_epoch().replace(tzinfo=None) + self.HOUR)  # type: ignore # https://github.com/python/typeshed/pull/1347
+
+    # This tests the LoggingCountStat branch of the code in do_delete_counts_at_hour.
+    # It is important that do_delete_counts_at_hour not delete any of the collected
+    # logging data!
+    def test_process_logging_stat(self):
+        # type: () -> None
+        end_time = self.TIME_ZERO
+
+        user_stat = LoggingCountStat('user stat', UserCount, CountStat.DAY)
+        stream_stat = LoggingCountStat('stream stat', StreamCount, CountStat.DAY)
+        realm_stat = LoggingCountStat('realm stat', RealmCount, CountStat.DAY)
+        user = self.create_user()
+        stream = self.create_stream_with_recipient()[0]
+        realm = self.default_realm
+        UserCount.objects.create(
+            user=user, realm=realm, property=user_stat.property, end_time=end_time, value=5)
+        StreamCount.objects.create(
+            stream=stream, realm=realm, property=stream_stat.property, end_time=end_time, value=5)
+        RealmCount.objects.create(
+            realm=realm, property=realm_stat.property, end_time=end_time, value=5)
+
+        # Normal run of process_count_stat
+        for stat in [user_stat, stream_stat, realm_stat]:
+            process_count_stat(stat, end_time)
+        self.assertTableState(UserCount, ['property', 'value'], [[user_stat.property, 5]])
+        self.assertTableState(StreamCount, ['property', 'value'], [[stream_stat.property, 5]])
+        self.assertTableState(RealmCount, ['property', 'value'],
+                              [[user_stat.property, 5], [stream_stat.property, 5], [realm_stat.property, 5]])
+        self.assertTableState(InstallationCount, ['property', 'value'],
+                              [[user_stat.property, 5], [stream_stat.property, 5], [realm_stat.property, 5]])
+
+        # Change the logged data and mark FillState as dirty
+        UserCount.objects.update(value=6)
+        StreamCount.objects.update(value=6)
+        RealmCount.objects.filter(property=realm_stat.property).update(value=6)
+        FillState.objects.update(state=FillState.STARTED)
+
+        # Check that the change propagated (and the collected data wasn't deleted)
+        for stat in [user_stat, stream_stat, realm_stat]:
+            process_count_stat(stat, end_time)
+        self.assertTableState(UserCount, ['property', 'value'], [[user_stat.property, 6]])
+        self.assertTableState(StreamCount, ['property', 'value'], [[stream_stat.property, 6]])
+        self.assertTableState(RealmCount, ['property', 'value'],
+                              [[user_stat.property, 6], [stream_stat.property, 6], [realm_stat.property, 6]])
+        self.assertTableState(InstallationCount, ['property', 'value'],
+                              [[user_stat.property, 6], [stream_stat.property, 6], [realm_stat.property, 6]])
+
+    def test_process_dependent_stat(self):
+        # type: () -> None
+        stat1 = self.make_dummy_count_stat('stat1')
+        stat2 = self.make_dummy_count_stat('stat2')
+        query = """INSERT INTO analytics_realmcount (realm_id, value, property, end_time)
+                   VALUES (%s, 1, '%s', %%%%(time_end)s)""" % (self.default_realm.id, 'stat3')
+        stat3 = DependentCountStat('stat3', sql_data_collector(RealmCount, query, None), CountStat.HOUR,
+                                   dependencies=['stat1', 'stat2'])
+        hour = [installation_epoch() + i*self.HOUR for i in range(5)]
+
+        # test when one dependency has been run, and the other hasn't
+        process_count_stat(stat1, hour[2])
+        process_count_stat(stat3, hour[1])
+        self.assertTableState(InstallationCount, ['property', 'end_time'],
+                              [['stat1', hour[1]], ['stat1', hour[2]]])
+        self.assertFillStateEquals(stat3, hour[0])
+
+        # test that we don't fill past the fill_to_time argument, even if
+        # dependencies have later last_successful_fill
+        process_count_stat(stat2, hour[3])
+        process_count_stat(stat3, hour[1])
+        self.assertTableState(InstallationCount, ['property', 'end_time'],
+                              [['stat1', hour[1]], ['stat1', hour[2]],
+                               ['stat2', hour[1]], ['stat2', hour[2]], ['stat2', hour[3]],
+                               ['stat3', hour[1]]])
+        self.assertFillStateEquals(stat3, hour[1])
+
+        # test that we don't fill past the dependency last_successful_fill times,
+        # even if fill_to_time is later
+        process_count_stat(stat3, hour[4])
+        self.assertTableState(InstallationCount, ['property', 'end_time'],
+                              [['stat1', hour[1]], ['stat1', hour[2]],
+                               ['stat2', hour[1]], ['stat2', hour[2]], ['stat2', hour[3]],
+                               ['stat3', hour[1]], ['stat3', hour[2]]])
+        self.assertFillStateEquals(stat3, hour[2])
+
+        # test daily dependent stat with hourly dependencies
+        query = """INSERT INTO analytics_realmcount (realm_id, value, property, end_time)
+                   VALUES (%s, 1, '%s', %%%%(time_end)s)""" % (self.default_realm.id, 'stat4')
+        stat4 = DependentCountStat('stat4', sql_data_collector(RealmCount, query, None), CountStat.DAY,
+                                   dependencies=['stat1', 'stat2'])
+        hour24 = installation_epoch() + 24*self.HOUR
+        hour25 = installation_epoch() + 25*self.HOUR
+        process_count_stat(stat1, hour25)
+        process_count_stat(stat2, hour25)
+        process_count_stat(stat4, hour25)
+        self.assertEqual(InstallationCount.objects.filter(property='stat4').count(), 1)
+        self.assertFillStateEquals(stat4, hour24)
 
 class TestCountStats(AnalyticsTestCase):
     def setUp(self):
@@ -201,7 +307,7 @@ class TestCountStats(AnalyticsTestCase):
         # the queries).
         self.second_realm = Realm.objects.create(
             string_id='second-realm', name='Second Realm',
-            domain='second.analytics', date_created=self.TIME_ZERO-2*self.DAY)
+            date_created=self.TIME_ZERO-2*self.DAY)
         for minutes_ago in [0, 1, 61, 60*24+1]:
             creation_time = self.TIME_ZERO - minutes_ago*self.MINUTE
             user = self.create_user(email='user-%s@second.analytics' % (minutes_ago,),
@@ -217,7 +323,7 @@ class TestCountStats(AnalyticsTestCase):
         # messages_* CountStats
         self.no_message_realm = Realm.objects.create(
             string_id='no-message-realm', name='No Message Realm',
-            domain='no.message', date_created=self.TIME_ZERO-2*self.DAY)
+            date_created=self.TIME_ZERO-2*self.DAY)
         self.create_user(realm=self.no_message_realm)
         self.create_stream_with_recipient(realm=self.no_message_realm)
         # This huddle should not show up anywhere
@@ -323,16 +429,19 @@ class TestCountStats(AnalyticsTestCase):
                                [2, 'private_stream', user2],
                                [2, 'public_stream', user1],
                                [1, 'public_stream', user2],
-                               [2, 'private_message', user1],
-                               [2, 'private_message', user2],
+                               [1, 'private_message', user1],
+                               [1, 'private_message', user2],
                                [1, 'private_message', user3],
+                               [1, 'huddle_message', user1],
+                               [1, 'huddle_message', user2],
                                [1, 'public_stream', self.hourly_user],
                                [1, 'public_stream', self.daily_user]])
         self.assertTableState(RealmCount, ['value', 'subgroup', 'realm'],
-                              [[3, 'private_stream'], [3, 'public_stream'], [5, 'private_message'],
-                               [2, 'public_stream', self.second_realm]])
+                              [[3, 'private_stream'], [3, 'public_stream'], [3, 'private_message'],
+                               [2, 'huddle_message'], [2, 'public_stream', self.second_realm]])
         self.assertTableState(InstallationCount, ['value', 'subgroup'],
-                              [[3, 'private_stream'], [5, 'public_stream'], [5, 'private_message']])
+                              [[3, 'private_stream'], [5, 'public_stream'], [3, 'private_message'],
+                               [2, 'huddle_message']])
         self.assertTableState(StreamCount, [], [])
 
     def test_messages_sent_to_recipients_with_same_id(self):
@@ -351,7 +460,8 @@ class TestCountStats(AnalyticsTestCase):
 
         do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
 
-        self.assertCountEquals(UserCount, 2, subgroup='private_message')
+        self.assertCountEquals(UserCount, 1, subgroup='private_message')
+        self.assertCountEquals(UserCount, 1, subgroup='huddle_message')
         self.assertCountEquals(UserCount, 1, subgroup='public_stream')
 
     def test_messages_sent_by_client(self):
@@ -377,7 +487,7 @@ class TestCountStats(AnalyticsTestCase):
         do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
 
         client2_id = str(client2.id)
-        website_client_id = str(get_client('website').id) # default for self.create_message
+        website_client_id = str(get_client('website').id)  # default for self.create_message
         self.assertTableState(UserCount, ['value', 'subgroup', 'user'],
                               [[2, website_client_id, user1],
                                [1, client2_id, user1], [2, client2_id, user2],
@@ -392,7 +502,7 @@ class TestCountStats(AnalyticsTestCase):
 
     def test_messages_sent_to_stream_by_is_bot(self):
         # type: () -> None
-        stat = COUNT_STATS['messages_sent_to_stream:is_bot:hour']
+        stat = COUNT_STATS['messages_in_stream:is_bot:day']
         self.current_property = stat.property
 
         bot = self.create_user(is_bot=True)
@@ -420,9 +530,520 @@ class TestCountStats(AnalyticsTestCase):
 
         self.assertTableState(StreamCount, ['value', 'subgroup', 'stream'],
                               [[2, 'false', stream1], [1, 'false', stream2], [2, 'true', stream2],
-                               # "hourly" stream, from TestCountStats.setUp
-                               [1, 'false', Stream.objects.get(name='stream 1')]])
+                               # "hourly" and "daily" stream, from TestCountStats.setUp
+                               [1, 'false', Stream.objects.get(name='stream 1')],
+                               [1, 'false', Stream.objects.get(name='stream 61')]])
         self.assertTableState(RealmCount, ['value', 'subgroup', 'realm'],
-                              [[3, 'false'], [2, 'true'], [1, 'false', self.second_realm]])
-        self.assertTableState(InstallationCount, ['value', 'subgroup'], [[4, 'false'], [2, 'true']])
+                              [[3, 'false'], [2, 'true'], [2, 'false', self.second_realm]])
+        self.assertTableState(InstallationCount, ['value', 'subgroup'], [[5, 'false'], [2, 'true']])
         self.assertTableState(UserCount, [], [])
+
+    def create_interval(self, user, start_offset, end_offset):
+        # type: (UserProfile, timedelta, timedelta) -> None
+        UserActivityInterval.objects.create(
+            user_profile=user, start=self.TIME_ZERO-start_offset,
+            end=self.TIME_ZERO-end_offset)
+
+    def test_15day_actives(self):
+        # type: () -> None
+        stat = COUNT_STATS['15day_actives::day']
+        self.current_property = stat.property
+
+        _15day = 15*self.DAY - UserActivityInterval.MIN_INTERVAL_LENGTH
+
+        # Outside time range, should not appear. Also tests upper boundary.
+        user1 = self.create_user()
+        self.create_interval(user1, _15day + self.DAY, _15day + timedelta(seconds=1))
+        self.create_interval(user1, timedelta(0), -self.HOUR)
+
+        # On lower boundary, should appear
+        user2 = self.create_user()
+        self.create_interval(user2, _15day + self.DAY, _15day)
+
+        # Multiple intervals, including one outside boundary
+        user3 = self.create_user()
+        self.create_interval(user3, 20*self.DAY, 19*self.DAY)
+        self.create_interval(user3, 20*self.HOUR, 19*self.HOUR)
+        self.create_interval(user3, 20*self.MINUTE, 19*self.MINUTE)
+
+        # Intervals crossing boundary
+        user4 = self.create_user()
+        self.create_interval(user4, 20*self.DAY, 10*self.DAY)
+        user5 = self.create_user()
+        self.create_interval(user5, self.MINUTE, -self.MINUTE)
+
+        # Interval subsuming time range
+        user6 = self.create_user()
+        self.create_interval(user6, 20*self.DAY, -2*self.DAY)
+
+        # Second realm
+        user7 = self.create_user(realm=self.second_realm)
+        self.create_interval(user7, 20*self.MINUTE, 19*self.MINUTE)
+
+        do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['value', 'user'],
+                              [[1, user2], [1, user3], [1, user4], [1, user5], [1, user6], [1, user7]])
+        self.assertTableState(RealmCount, ['value', 'realm'],
+                              [[5, self.default_realm], [1, self.second_realm]])
+        self.assertTableState(InstallationCount, ['value'], [[6]])
+        self.assertTableState(StreamCount, [], [])
+
+    def test_minutes_active(self):
+        # type: () -> None
+        stat = COUNT_STATS['minutes_active::day']
+        self.current_property = stat.property
+
+        # Outside time range, should not appear. Also testing for intervals
+        # starting and ending on boundary
+        user1 = self.create_user()
+        self.create_interval(user1, 25*self.HOUR, self.DAY)
+        self.create_interval(user1, timedelta(0), -self.HOUR)
+
+        # Multiple intervals, including one outside boundary
+        user2 = self.create_user()
+        self.create_interval(user2, 20*self.DAY, 19*self.DAY)
+        self.create_interval(user2, 20*self.HOUR, 19*self.HOUR)
+        self.create_interval(user2, 20*self.MINUTE, 19*self.MINUTE)
+
+        # Intervals crossing boundary
+        user3 = self.create_user()
+        self.create_interval(user3, 25*self.HOUR, 22*self.HOUR)
+        self.create_interval(user3, self.MINUTE, -self.MINUTE)
+
+        # Interval subsuming time range
+        user4 = self.create_user()
+        self.create_interval(user4, 2*self.DAY, -2*self.DAY)
+
+        # Less than 60 seconds, should not appear
+        user5 = self.create_user()
+        self.create_interval(user5, self.MINUTE, timedelta(seconds=30))
+        self.create_interval(user5, timedelta(seconds=20), timedelta(seconds=10))
+
+        # Second realm
+        user6 = self.create_user(realm=self.second_realm)
+        self.create_interval(user6, 20*self.MINUTE, 19*self.MINUTE)
+
+        do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['value', 'user'],
+                              [[61, user2], [121, user3], [24*60, user4], [1, user6]])
+        self.assertTableState(RealmCount, ['value', 'realm'],
+                              [[61 + 121 + 24*60, self.default_realm], [1, self.second_realm]])
+        self.assertTableState(InstallationCount, ['value'], [[61 + 121 + 24*60 + 1]])
+        self.assertTableState(StreamCount, [], [])
+
+class TestDoAggregateToSummaryTable(AnalyticsTestCase):
+    # do_aggregate_to_summary_table is mostly tested by the end to end
+    # nature of the tests in TestCountStats. But want to highlight one
+    # feature important for keeping the size of the analytics tables small,
+    # which is that if there is no relevant data in the table being
+    # aggregated, the aggregation table doesn't get a row with value 0.
+    def test_no_aggregated_zeros(self):
+        # type: () -> None
+        stat = LoggingCountStat('test stat', UserCount, CountStat.HOUR)
+        do_aggregate_to_summary_table(stat, self.TIME_ZERO)
+        self.assertFalse(RealmCount.objects.exists())
+        self.assertFalse(InstallationCount.objects.exists())
+
+class TestDoIncrementLoggingStat(AnalyticsTestCase):
+    def test_table_and_id_args(self):
+        # type: () -> None
+        # For realms, streams, and users, tests that the new rows are going to
+        # the appropriate *Count table, and that using a different zerver_object
+        # results in a new row being created
+        self.current_property = 'test'
+        second_realm = Realm.objects.create(string_id='moo', name='moo')
+        stat = LoggingCountStat('test', RealmCount, CountStat.DAY)
+        do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
+        do_increment_logging_stat(second_realm, stat, None, self.TIME_ZERO)
+        self.assertTableState(RealmCount, ['realm'], [[self.default_realm], [second_realm]])
+
+        user1 = self.create_user()
+        user2 = self.create_user()
+        stat = LoggingCountStat('test', UserCount, CountStat.DAY)
+        do_increment_logging_stat(user1, stat, None, self.TIME_ZERO)
+        do_increment_logging_stat(user2, stat, None, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['user'], [[user1], [user2]])
+
+        stream1 = self.create_stream_with_recipient()[0]
+        stream2 = self.create_stream_with_recipient()[0]
+        stat = LoggingCountStat('test', StreamCount, CountStat.DAY)
+        do_increment_logging_stat(stream1, stat, None, self.TIME_ZERO)
+        do_increment_logging_stat(stream2, stat, None, self.TIME_ZERO)
+        self.assertTableState(StreamCount, ['stream'], [[stream1], [stream2]])
+
+    def test_frequency(self):
+        # type: () -> None
+        times = [self.TIME_ZERO - self.MINUTE*i for i in [0, 1, 61, 24*60+1]]
+
+        stat = LoggingCountStat('day test', RealmCount, CountStat.DAY)
+        for time_ in times:
+            do_increment_logging_stat(self.default_realm, stat, None, time_)
+        stat = LoggingCountStat('hour test', RealmCount, CountStat.HOUR)
+        for time_ in times:
+            do_increment_logging_stat(self.default_realm, stat, None, time_)
+
+        self.assertTableState(RealmCount, ['value', 'property', 'end_time'],
+                              [[3, 'day test', self.TIME_ZERO],
+                               [1, 'day test', self.TIME_ZERO - self.DAY],
+                               [2, 'hour test', self.TIME_ZERO],
+                               [1, 'hour test', self.TIME_LAST_HOUR],
+                               [1, 'hour test', self.TIME_ZERO - self.DAY]])
+
+    def test_get_or_create(self):
+        # type: () -> None
+        stat = LoggingCountStat('test', RealmCount, CountStat.HOUR)
+        # All these should trigger the create part of get_or_create.
+        # property is tested in test_frequency, and id_args are tested in test_id_args,
+        # so this only tests a new subgroup and end_time
+        do_increment_logging_stat(self.default_realm, stat, 'subgroup1', self.TIME_ZERO)
+        do_increment_logging_stat(self.default_realm, stat, 'subgroup2', self.TIME_ZERO)
+        do_increment_logging_stat(self.default_realm, stat, 'subgroup1', self.TIME_LAST_HOUR)
+        self.current_property = 'test'
+        self.assertTableState(RealmCount, ['value', 'subgroup', 'end_time'],
+                              [[1, 'subgroup1', self.TIME_ZERO], [1, 'subgroup2', self.TIME_ZERO],
+                              [1, 'subgroup1', self.TIME_LAST_HOUR]])
+        # This should trigger the get part of get_or_create
+        do_increment_logging_stat(self.default_realm, stat, 'subgroup1', self.TIME_ZERO)
+        self.assertTableState(RealmCount, ['value', 'subgroup', 'end_time'],
+                              [[2, 'subgroup1', self.TIME_ZERO], [1, 'subgroup2', self.TIME_ZERO],
+                              [1, 'subgroup1', self.TIME_LAST_HOUR]])
+
+    def test_increment(self):
+        # type: () -> None
+        stat = LoggingCountStat('test', RealmCount, CountStat.DAY)
+        self.current_property = 'test'
+        do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO, increment=-1)
+        self.assertTableState(RealmCount, ['value'], [[-1]])
+        do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO, increment=3)
+        self.assertTableState(RealmCount, ['value'], [[2]])
+        do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
+        self.assertTableState(RealmCount, ['value'], [[3]])
+
+class TestLoggingCountStats(AnalyticsTestCase):
+    def test_aggregation(self):
+        # type: () -> None
+        stat = LoggingCountStat('realm test', RealmCount, CountStat.DAY)
+        do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
+        process_count_stat(stat, self.TIME_ZERO)
+
+        user = self.create_user()
+        stat = LoggingCountStat('user test', UserCount, CountStat.DAY)
+        do_increment_logging_stat(user, stat, None, self.TIME_ZERO)
+        process_count_stat(stat, self.TIME_ZERO)
+
+        stream = self.create_stream_with_recipient()[0]
+        stat = LoggingCountStat('stream test', StreamCount, CountStat.DAY)
+        do_increment_logging_stat(stream, stat, None, self.TIME_ZERO)
+        process_count_stat(stat, self.TIME_ZERO)
+
+        self.assertTableState(InstallationCount, ['property', 'value'],
+                              [['realm test', 1], ['user test', 1], ['stream test', 1]])
+        self.assertTableState(RealmCount, ['property', 'value'],
+                              [['realm test', 1], ['user test', 1], ['stream test', 1]])
+        self.assertTableState(UserCount, ['property', 'value'], [['user test', 1]])
+        self.assertTableState(StreamCount, ['property', 'value'], [['stream test', 1]])
+
+    def test_active_users_log_by_is_bot(self):
+        # type: () -> None
+        property = 'active_users_log:is_bot:day'
+        user = do_create_user('email', 'password', self.default_realm, 'full_name', 'short_name')
+        self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
+                         .aggregate(Sum('value'))['value__sum'])
+        do_deactivate_user(user)
+        self.assertEqual(0, RealmCount.objects.filter(property=property, subgroup=False)
+                         .aggregate(Sum('value'))['value__sum'])
+        do_activate_user(user)
+        self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
+                         .aggregate(Sum('value'))['value__sum'])
+        do_deactivate_user(user)
+        self.assertEqual(0, RealmCount.objects.filter(property=property, subgroup=False)
+                         .aggregate(Sum('value'))['value__sum'])
+        do_reactivate_user(user)
+        self.assertEqual(1, RealmCount.objects.filter(property=property, subgroup=False)
+                         .aggregate(Sum('value'))['value__sum'])
+
+class TestDeleteStats(AnalyticsTestCase):
+    def test_do_drop_all_analytics_tables(self):
+        # type: () -> None
+        user = self.create_user()
+        stream = self.create_stream_with_recipient()[0]
+        count_args = {'property': 'test', 'end_time': self.TIME_ZERO, 'value': 10}
+
+        UserCount.objects.create(user=user, realm=user.realm, **count_args)
+        StreamCount.objects.create(stream=stream, realm=stream.realm, **count_args)
+        RealmCount.objects.create(realm=user.realm, **count_args)
+        InstallationCount.objects.create(**count_args)
+        FillState.objects.create(property='test', end_time=self.TIME_ZERO, state=FillState.DONE)
+        Anomaly.objects.create(info='test anomaly')
+
+        analytics = apps.get_app_config('analytics')
+        for table in list(analytics.models.values()):
+            self.assertTrue(table.objects.exists())
+
+        do_drop_all_analytics_tables()
+        for table in list(analytics.models.values()):
+            self.assertFalse(table.objects.exists())
+
+class TestActiveUsersAudit(AnalyticsTestCase):
+    def setUp(self):
+        # type: () -> None
+        super(TestActiveUsersAudit, self).setUp()
+        self.user = self.create_user()
+        self.stat = COUNT_STATS['active_users_audit:is_bot:day']
+        self.current_property = self.stat.property
+
+    def add_event(self, event_type, days_offset, user=None):
+        # type: (str, float, Optional[UserProfile]) -> None
+        hours_offset = int(24*days_offset)
+        if user is None:
+            user = self.user
+        RealmAuditLog.objects.create(
+            realm=user.realm, modified_user=user, event_type=event_type,
+            event_time=self.TIME_ZERO - hours_offset*self.HOUR)
+
+    def test_user_deactivated_in_future(self):
+        # type: () -> None
+        self.add_event('user_created', 1)
+        self.add_event('user_deactivated', 0)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup'], [['false']])
+
+    def test_user_reactivated_in_future(self):
+        # type: () -> None
+        self.add_event('user_deactivated', 1)
+        self.add_event('user_reactivated', 0)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, [], [])
+
+    def test_user_active_then_deactivated_same_day(self):
+        # type: () -> None
+        self.add_event('user_created', 1)
+        self.add_event('user_deactivated', .5)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, [], [])
+
+    def test_user_unactive_then_activated_same_day(self):
+        # type: () -> None
+        self.add_event('user_deactivated', 1)
+        self.add_event('user_reactivated', .5)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup'], [['false']])
+
+    # Arguably these next two tests are duplicates of the _in_future tests, but are
+    # a guard against future refactorings where they may no longer be duplicates
+    def test_user_active_then_deactivated_with_day_gap(self):
+        # type: () -> None
+        self.add_event('user_created', 2)
+        self.add_event('user_deactivated', 1)
+        process_count_stat(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup', 'end_time'],
+                              [['false', self.TIME_ZERO - self.DAY]])
+
+    def test_user_deactivated_then_reactivated_with_day_gap(self):
+        # type: () -> None
+        self.add_event('user_deactivated', 2)
+        self.add_event('user_reactivated', 1)
+        process_count_stat(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup'], [['false']])
+
+    def test_event_types(self):
+        # type: () -> None
+        self.add_event('user_created', 4)
+        self.add_event('user_deactivated', 3)
+        self.add_event('user_activated', 2)
+        self.add_event('user_reactivated', 1)
+        for i in range(4):
+            do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO - i*self.DAY)
+        self.assertTableState(UserCount, ['subgroup', 'end_time'],
+                              [['false', self.TIME_ZERO - i*self.DAY] for i in [3, 1, 0]])
+
+    # Also tests that aggregation to RealmCount and InstallationCount is
+    # being done, and that we're storing the user correctly in UserCount
+    def test_multiple_users_realms_and_bots(self):
+        # type: () -> None
+        user1 = self.create_user()
+        user2 = self.create_user()
+        second_realm = Realm.objects.create(string_id='moo', name='moo')
+        user3 = self.create_user(realm=second_realm)
+        user4 = self.create_user(realm=second_realm, is_bot=True)
+        for user in [user1, user2, user3, user4]:
+            self.add_event('user_created', 1, user=user)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup', 'user'],
+                              [['false', user1], ['false', user2], ['false', user3], ['true', user4]])
+        self.assertTableState(RealmCount, ['value', 'subgroup', 'realm'],
+                              [[2, 'false', self.default_realm], [1, 'false', second_realm],
+                               [1, 'true', second_realm]])
+        self.assertTableState(InstallationCount, ['value', 'subgroup'], [[3, 'false'], [1, 'true']])
+        self.assertTableState(StreamCount, [], [])
+
+    # Not that interesting a test if you look at the SQL query at hand, but
+    # almost all other CountStats have a start_date, so guarding against a
+    # refactoring that adds that in.
+    # Also tests the slightly more end-to-end process_count_stat rather than
+    # do_fill_count_stat_at_hour. E.g. if one changes self.stat.frequency to
+    # CountStat.HOUR from CountStat.DAY, this will fail, while many of the
+    # tests above will not.
+    def test_update_from_two_days_ago(self):
+        # type: () -> None
+        self.add_event('user_created', 2)
+        process_count_stat(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup', 'end_time'],
+                              [['false', self.TIME_ZERO], ['false', self.TIME_ZERO-self.DAY]])
+
+    # User with no relevant activity could happen e.g. for a system bot that
+    # doesn't go through do_create_user. Mainly just want to make sure that
+    # that situation doesn't throw an error.
+    def test_empty_realm_or_user_with_no_relevant_activity(self):
+        # type: () -> None
+        self.add_event('unrelated', 1)
+        self.create_user()  # also test a user with no RealmAuditLog entries
+        Realm.objects.create(string_id='moo', name='moo')
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, [], [])
+
+    def test_max_audit_entry_is_unrelated(self):
+        # type: () -> None
+        self.add_event('user_created', 1)
+        self.add_event('unrelated', .5)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup'], [['false']])
+
+    # Simultaneous related audit entries should not be allowed, and so not testing for that.
+    def test_simultaneous_unrelated_audit_entry(self):
+        # type: () -> None
+        self.add_event('user_created', 1)
+        self.add_event('unrelated', 1)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['subgroup'], [['false']])
+
+    def test_simultaneous_max_audit_entries_of_different_users(self):
+        # type: () -> None
+        user1 = self.create_user()
+        user2 = self.create_user()
+        user3 = self.create_user()
+        self.add_event('user_created', .5, user=user1)
+        self.add_event('user_created', .5, user=user2)
+        self.add_event('user_created', 1, user=user3)
+        self.add_event('user_deactivated', .5, user=user3)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(UserCount, ['user', 'subgroup'],
+                              [[user1, 'false'], [user2, 'false']])
+
+    def test_end_to_end_with_actions_dot_py(self):
+        # type: () -> None
+        user1 = do_create_user('email1', 'password', self.default_realm, 'full_name', 'short_name')
+        user2 = do_create_user('email2', 'password', self.default_realm, 'full_name', 'short_name')
+        user3 = do_create_user('email3', 'password', self.default_realm, 'full_name', 'short_name')
+        user4 = do_create_user('email4', 'password', self.default_realm, 'full_name', 'short_name')
+        do_deactivate_user(user2)
+        do_activate_user(user3)
+        do_reactivate_user(user4)
+        end_time = floor_to_day(timezone_now()) + self.DAY
+        do_fill_count_stat_at_hour(self.stat, end_time)
+        for user in [user1, user3, user4]:
+            self.assertTrue(UserCount.objects.filter(
+                user=user, property=self.current_property, subgroup='false',
+                end_time=end_time, value=1).exists())
+        self.assertFalse(UserCount.objects.filter(user=user2).exists())
+
+class TestRealmActiveHumans(AnalyticsTestCase):
+    def setUp(self):
+        # type: () -> None
+        super(TestRealmActiveHumans, self).setUp()
+        self.stat = COUNT_STATS['realm_active_humans::day']
+        self.current_property = self.stat.property
+
+    def mark_audit_active(self, user, end_time=None):
+        # type: (UserProfile, Optional[datetime]) -> None
+        if end_time is None:
+            end_time = self.TIME_ZERO
+        UserCount.objects.create(
+            user=user, realm=user.realm, property='active_users_audit:is_bot:day',
+            subgroup=ujson.dumps(user.is_bot), end_time=end_time, value=1)
+
+    def mark_15day_active(self, user, end_time=None):
+        # type: (UserProfile, Optional[datetime]) -> None
+        if end_time is None:
+            end_time = self.TIME_ZERO
+        UserCount.objects.create(
+            user=user, realm=user.realm, property='15day_actives::day',
+            end_time=end_time, value=1)
+
+    def test_basic_boolean_logic(self):
+        # type: () -> None
+        user = self.create_user()
+        self.mark_audit_active(user, end_time=self.TIME_ZERO - self.DAY)
+        self.mark_15day_active(user, end_time=self.TIME_ZERO)
+        self.mark_audit_active(user, end_time=self.TIME_ZERO + self.DAY)
+        self.mark_15day_active(user, end_time=self.TIME_ZERO + self.DAY)
+
+        for i in [-1, 0, 1]:
+            do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i*self.DAY)
+        self.assertTableState(RealmCount, ['value', 'end_time'], [[1, self.TIME_ZERO + self.DAY]])
+
+    def test_bots_not_counted(self):
+        # type: () -> None
+        bot = self.create_user(is_bot=True)
+        self.mark_audit_active(bot)
+        self.mark_15day_active(bot)
+        do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(RealmCount, [], [])
+
+    def test_multiple_users_realms_and_times(self):
+        # type: () -> None
+        user1 = self.create_user()
+        user2 = self.create_user()
+        second_realm = Realm.objects.create(string_id='second', name='second')
+        user3 = self.create_user(realm=second_realm)
+        user4 = self.create_user(realm=second_realm)
+        user5 = self.create_user(realm=second_realm)
+
+        for user in [user1, user2, user3, user4, user5]:
+            self.mark_audit_active(user)
+            self.mark_15day_active(user)
+        for user in [user1, user3, user4]:
+            self.mark_audit_active(user, end_time=self.TIME_ZERO - self.DAY)
+            self.mark_15day_active(user, end_time=self.TIME_ZERO - self.DAY)
+
+        for i in [-1, 0, 1]:
+            do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i*self.DAY)
+        self.assertTableState(RealmCount, ['value', 'realm', 'end_time'],
+                              [[2, self.default_realm, self.TIME_ZERO],
+                               [3, second_realm, self.TIME_ZERO],
+                               [1, self.default_realm, self.TIME_ZERO - self.DAY],
+                               [2, second_realm, self.TIME_ZERO - self.DAY]])
+
+        # Check that adding spurious entries doesn't make a difference
+        self.mark_audit_active(user1, end_time=self.TIME_ZERO + self.DAY)
+        self.mark_15day_active(user2, end_time=self.TIME_ZERO + self.DAY)
+        self.mark_15day_active(user2, end_time=self.TIME_ZERO - self.DAY)
+        self.create_user()
+        third_realm = Realm.objects.create(string_id='third', name='third')
+        self.create_user(realm=third_realm)
+
+        RealmCount.objects.all().delete()
+        for i in [-1, 0, 1]:
+            do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i*self.DAY)
+        self.assertTableState(RealmCount, ['value', 'realm', 'end_time'],
+                              [[2, self.default_realm, self.TIME_ZERO],
+                               [3, second_realm, self.TIME_ZERO],
+                               [1, self.default_realm, self.TIME_ZERO - self.DAY],
+                               [2, second_realm, self.TIME_ZERO - self.DAY]])
+
+    def test_end_to_end(self):
+        # type: () -> None
+        user1 = do_create_user('email1', 'password', self.default_realm, 'full_name', 'short_name')
+        user2 = do_create_user('email2', 'password', self.default_realm, 'full_name', 'short_name')
+        do_create_user('email3', 'password', self.default_realm, 'full_name', 'short_name')
+        time_zero = floor_to_day(timezone_now()) + self.DAY
+        update_user_activity_interval(user1, time_zero)
+        update_user_activity_interval(user2, time_zero)
+        do_deactivate_user(user2)
+        for property in ['active_users_audit:is_bot:day', '15day_actives::day',
+                         'realm_active_humans::day']:
+            FillState.objects.create(property=property, state=FillState.DONE, end_time=time_zero)
+            process_count_stat(COUNT_STATS[property], time_zero+self.DAY)
+        self.assertEqual(RealmCount.objects.filter(
+            property='realm_active_humans::day', end_time=time_zero+self.DAY, value=1).count(), 1)
+        self.assertEqual(RealmCount.objects.filter(property='realm_active_humans::day').count(), 1)

analytics/tests/test_fixtures.py

@@ -0,0 +1,31 @@
+from zerver.lib.test_classes import ZulipTestCase
+
+from analytics.lib.counts import CountStat
+from analytics.lib.fixtures import generate_time_series_data
+
+# A very light test suite; the code being tested is not run in production.
+class TestFixtures(ZulipTestCase):
+    def test_deterministic_settings(self):
+        # type: () -> None
+        # test basic business_hour / non_business_hour calculation
+        # test we get an array of the right length with frequency=CountStat.DAY
+        data = generate_time_series_data(
+            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
+        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
+
+        data = generate_time_series_data(
+            days=1, business_hours_base=2000, non_business_hours_base=1500,
+            growth=2, spikiness=0, frequency=CountStat.HOUR)
+        # test we get an array of the right length with frequency=CountStat.HOUR
+        self.assertEqual(len(data), 24)
+        # test that growth doesn't affect the first data point
+        self.assertEqual(data[0], 2000)
+        # test that the last data point is growth times what it otherwise would be
+        self.assertEqual(data[-1], 1500*2)
+
+        # test autocorrelation == 1, since that's the easiest value to test
+        data = generate_time_series_data(
+            days=1, business_hours_base=2000, non_business_hours_base=2000,
+            autocorrelation=1, frequency=CountStat.HOUR)
+        self.assertEqual(data[0], data[1])
+        self.assertEqual(data[0], data[-1])

analytics/tests/test_views.py

@@ -1,18 +1,283 @@
-from django.utils.timezone import get_fixed_timezone
-from zerver.lib.test_classes import ZulipTestCase
+from __future__ import absolute_import
 
-from analytics.lib.counts import CountStat
+from django.utils.timezone import get_fixed_timezone, utc
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, \
+    datetime_to_timestamp
+from zerver.models import Realm, UserProfile, Client, get_realm, \
+    get_user_profile_by_email
+
+from analytics.lib.counts import CountStat, COUNT_STATS
 from analytics.lib.time_utils import time_range
-from analytics.views import rewrite_client_arrays
+from analytics.models import RealmCount, UserCount, BaseCount, \
+    FillState, last_successful_fill
+from analytics.views import stats, get_chart_data, sort_by_totals, \
+    sort_client_labels, rewrite_client_arrays
 
 from datetime import datetime, timedelta
+import mock
+import ujson
+
+from six.moves import range
+from typing import List, Dict, Optional
+
+class TestStatsEndpoint(ZulipTestCase):
+    def test_stats(self):
+        # type: () -> None
+        self.user = get_user_profile_by_email('hamlet@zulip.com')
+        self.login(self.user.email)
+        result = self.client_get('/stats')
+        self.assertEqual(result.status_code, 200)
+        # Check that we get something back
+        self.assert_in_response("Zulip Analytics for", result)
+
+class TestGetChartData(ZulipTestCase):
+    def setUp(self):
+        # type: () -> None
+        self.realm = get_realm('zulip')
+        self.user = get_user_profile_by_email('hamlet@zulip.com')
+        self.login(self.user.email)
+        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
+                               for i in range(4)]
+        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
+                              for i in range(4)]
+
+    def data(self, i):
+        # type: (int) -> List[int]
+        return [0, 0, i, 0]
+
+    def insert_data(self, stat, realm_subgroups, user_subgroups):
+        # type: (CountStat, List[Optional[str]], List[str]) -> None
+        if stat.frequency == CountStat.HOUR:
+            insert_time = self.end_times_hour[2]
+            fill_time = self.end_times_hour[-1]
+        if stat.frequency == CountStat.DAY:
+            insert_time = self.end_times_day[2]
+            fill_time = self.end_times_day[-1]
+
+        RealmCount.objects.bulk_create([
+            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
+                       value=100+i, realm=self.realm)
+            for i, subgroup in enumerate(realm_subgroups)])
+        UserCount.objects.bulk_create([
+            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
+                      value=200+i, realm=self.realm, user=self.user)
+            for i, subgroup in enumerate(user_subgroups)])
+        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
+
+    def test_number_of_humans(self):
+        # type: () -> None
+        stat = COUNT_STATS['realm_active_humans::day']
+        self.insert_data(stat, [None], [])
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data, {
+            'msg': '',
+            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
+            'frequency': CountStat.DAY,
+            'realm': {'human': self.data(100)},
+            'display_order': None,
+            'result': 'success',
+        })
+
+    def test_messages_sent_over_time(self):
+        # type: () -> None
+        stat = COUNT_STATS['messages_sent:is_bot:hour']
+        self.insert_data(stat, ['true', 'false'], ['false'])
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_over_time'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data, {
+            'msg': '',
+            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
+            'frequency': CountStat.HOUR,
+            'realm': {'bot': self.data(100), 'human': self.data(101)},
+            'user': {'bot': self.data(0), 'human': self.data(200)},
+            'display_order': None,
+            'result': 'success',
+        })
+
+    def test_messages_sent_by_message_type(self):
+        # type: () -> None
+        stat = COUNT_STATS['messages_sent:message_type:day']
+        self.insert_data(stat, ['public_stream', 'private_message'],
+                         ['public_stream', 'private_stream'])
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_by_message_type'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data, {
+            'msg': '',
+            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
+            'frequency': CountStat.DAY,
+            'realm': {'Public streams': self.data(100), 'Private streams': self.data(0),
+                      'Private messages': self.data(101), 'Group private messages': self.data(0)},
+            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
+                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
+            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
+            'result': 'success',
+        })
+
+    def test_messages_sent_by_client(self):
+        # type: () -> None
+        stat = COUNT_STATS['messages_sent:client:day']
+        client1 = Client.objects.create(name='client 1')
+        client2 = Client.objects.create(name='client 2')
+        client3 = Client.objects.create(name='client 3')
+        client4 = Client.objects.create(name='client 4')
+        self.insert_data(stat, [client4.id, client3.id, client2.id],
+                         [client3.id, client1.id])
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_by_client'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data, {
+            'msg': '',
+            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
+            'frequency': CountStat.DAY,
+            'realm': {'client 4': self.data(100), 'client 3': self.data(101),
+                      'client 2': self.data(102)},
+            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
+            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
+            'result': 'success',
+        })
+
+    def test_include_empty_subgroups(self):
+        # type: () -> None
+        FillState.objects.create(
+            property='realm_active_humans::day', end_time=self.end_times_day[0], state=FillState.DONE)
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['realm'], {'human': [0]})
+        self.assertFalse('user' in data)
+
+        FillState.objects.create(
+            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0], state=FillState.DONE)
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_over_time'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['realm'], {'human': [0], 'bot': [0]})
+        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
+
+        FillState.objects.create(
+            property='messages_sent:message_type:day', end_time=self.end_times_day[0], state=FillState.DONE)
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_by_message_type'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['realm'], {
+            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
+        self.assertEqual(data['user'], {
+            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
+
+        FillState.objects.create(
+            property='messages_sent:client:day', end_time=self.end_times_day[0], state=FillState.DONE)
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'messages_sent_by_client'})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['realm'], {})
+        self.assertEqual(data['user'], {})
+
+    def test_start_and_end(self):
+        # type: () -> None
+        stat = COUNT_STATS['realm_active_humans::day']
+        self.insert_data(stat, [None], [])
+        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
+
+        # valid start and end
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans',
+                                  'start': end_time_timestamps[1],
+                                  'end': end_time_timestamps[2]})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
+        self.assertEqual(data['realm'], {'human': [0, 100]})
+
+        # start later then end
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans',
+                                  'start': end_time_timestamps[2],
+                                  'end': end_time_timestamps[1]})
+        self.assert_json_error_contains(result, 'Start time is later than')
+
+    def test_min_length(self):
+        # type: () -> None
+        stat = COUNT_STATS['realm_active_humans::day']
+        self.insert_data(stat, [None], [])
+        # test min_length is too short to change anything
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans',
+                                  'min_length': 2})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
+        self.assertEqual(data['realm'], {'human': self.data(100)})
+        # test min_length larger than filled data
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'number_of_humans',
+                                  'min_length': 5})
+        self.assert_json_success(result)
+        data = ujson.loads(result.content)
+        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
+        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
+        self.assertEqual(data['realm'], {'human': [0]+self.data(100)})
+
+    def test_non_existent_chart(self):
+        # type: () -> None
+        result = self.client_get('/json/analytics/chart_data',
+                                 {'chart_name': 'does_not_exist'})
+        self.assert_json_error_contains(result, 'Unknown chart name')
+
+    def test_analytics_not_running(self):
+        # type: () -> None
+        # try to get data for a valid chart, but before we've put anything in the database
+        # (e.g. before update_analytics_counts has been run)
+        with mock.patch('logging.warning'):
+            result = self.client_get('/json/analytics/chart_data',
+                                     {'chart_name': 'number_of_humans'})
+        self.assert_json_error_contains(result, 'No analytics data available')
+
+class TestGetChartDataHelpers(ZulipTestCase):
+    # last_successful_fill is in analytics/models.py, but get_chart_data is
+    # the only function that uses it at the moment
+    def test_last_successful_fill(self):
+        # type: () -> None
+        self.assertIsNone(last_successful_fill('non-existant'))
+        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
+        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
+        fillstate = FillState.objects.create(property='property', end_time=a_time,
+                                             state=FillState.DONE)
+        self.assertEqual(last_successful_fill('property'), a_time)
+        fillstate.state = FillState.STARTED
+        fillstate.save()
+        self.assertEqual(last_successful_fill('property'), one_hour_before)
+
+    def test_sort_by_totals(self):
+        # type: () -> None
+        empty = []  # type: List[int]
+        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
+        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
+
+    def test_sort_client_labels(self):
+        # type: () -> None
+        data = {'realm': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
+                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
+        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
 
 class TestTimeRange(ZulipTestCase):
     def test_time_range(self):
         # type: () -> None
         HOUR = timedelta(hours=1)
         DAY = timedelta(days=1)
-        TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC
+        TZINFO = get_fixed_timezone(-100)  # 100 minutes west of UTC
 
         # Using 22:59 so that converting to UTC and applying floor_to_{hour,day} do not commute
         a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)

analytics/views.py

@@ -1,45 +1,49 @@
 from __future__ import absolute_import, division
 
+from django.conf import settings
 from django.core import urlresolvers
 from django.db import connection
 from django.db.models import Sum
 from django.db.models.query import QuerySet
 from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from django.template import RequestContext, loader
-from django.utils import timezone
+from django.utils.timezone import now as timezone_now
 from django.utils.translation import ugettext as _
+from django.shortcuts import render
 from jinja2 import Markup as mark_safe
 
 from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS
 from analytics.lib.time_utils import time_range
 from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount
+    UserCount, StreamCount, last_successful_fill
 
-from zerver.decorator import has_request_variables, REQ, zulip_internal, \
+from zerver.decorator import has_request_variables, REQ, require_server_admin, \
     zulip_login_required, to_non_negative_int, to_utc_datetime
 from zerver.lib.request import JsonableError
 from zerver.lib.response import json_success
 from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, timestamp_to_datetime
 from zerver.models import Realm, UserProfile, UserActivity, \
     UserActivityInterval, Client
-from zproject.jinja2 import render_to_response
 
 from collections import defaultdict
 from datetime import datetime, timedelta
 import itertools
 import json
+import logging
 import pytz
 import re
 import time
 
 from six.moves import filter, map, range, zip
-from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Type, \
-    Union, Text
+from typing import Any, Callable, Dict, List, Optional, Set, Text, \
+    Tuple, Type, Union
 
 @zulip_login_required
 def stats(request):
     # type: (HttpRequest) -> HttpResponse
-    return render_to_response('analytics/stats.html')
+    return render(request,
+                  'analytics/stats.html',
+                  context=dict(realm_name = request.user.realm.name))
 
 @has_request_variables
 def get_chart_data(request, user_profile, chart_name=REQ(),
@@ -47,54 +51,94 @@ def get_chart_data(request, user_profile, chart_name=REQ(),
                    start=REQ(converter=to_utc_datetime, default=None),
                    end=REQ(converter=to_utc_datetime, default=None)):
     # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse
-    realm = user_profile.realm
-    # These are implicitly relying on realm.date_created and timezone.now being in UTC.
-    if start is None:
-        start = realm.date_created
-    if end is None:
-        end = timezone.now()
-    if start > end:
-        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
-                            {'start': start, 'end': end})
-
     if chart_name == 'number_of_humans':
-        stat = COUNT_STATS['active_users:is_bot:day']
+        stat = COUNT_STATS['realm_active_humans::day']
         tables = [RealmCount]
-        subgroups = ['false', 'true']
-        labels = ['human', 'bot']
+        subgroup_to_label = {None: 'human'}  # type: Dict[Optional[str], str]
+        labels_sort_function = None
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_over_time':
         stat = COUNT_STATS['messages_sent:is_bot:hour']
-        tables = [RealmCount]
-        subgroups = ['false', 'true']
-        labels = ['human', 'bot']
+        tables = [RealmCount, UserCount]
+        subgroup_to_label = {'false': 'human', 'true': 'bot'}
+        labels_sort_function = None
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_by_message_type':
         stat = COUNT_STATS['messages_sent:message_type:day']
         tables = [RealmCount, UserCount]
-        subgroups = ['public_stream', 'private_stream', 'private_message']
-        labels = None
+        subgroup_to_label = {'public_stream': 'Public streams',
+                             'private_stream': 'Private streams',
+                             'private_message': 'Private messages',
+                             'huddle_message': 'Group private messages'}
+        labels_sort_function = lambda data: sort_by_totals(data['realm'])
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_by_client':
         stat = COUNT_STATS['messages_sent:client:day']
         tables = [RealmCount, UserCount]
-        subgroups = [str(x) for x in Client.objects.values_list('id', flat=True).order_by('id')]
-        labels = list(Client.objects.values_list('name', flat=True).order_by('id'))
+        # Note that the labels are further re-written by client_label_map
+        subgroup_to_label = {str(id): name for id, name in Client.objects.values_list('id', 'name')}
+        labels_sort_function = sort_client_labels
         include_empty_subgroups = False
     else:
         raise JsonableError(_("Unknown chart name: %s") % (chart_name,))
 
+    # Most likely someone using our API endpoint. The /stats page does not
+    # pass a start or end in its requests.
+    if start is not None and end is not None and start > end:
+        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
+                            {'start': start, 'end': end})
+
+    realm = user_profile.realm
+    if start is None:
+        start = realm.date_created
+    if end is None:
+        end = last_successful_fill(stat.property)
+    if end is None or start > end:
+        logging.warning("User from realm %s attempted to access /stats, but the computed "
+                        "start time: %s (creation time of realm) is later than the computed "
+                        "end time: %s (last successful analytics update). Is the "
+                        "analytics cron job running?" % (realm.string_id, start, end))
+        raise JsonableError(_("No analytics data available. Please contact your server administrator."))
+
     end_times = time_range(start, end, stat.frequency, min_length)
-    data = {'end_times': end_times, 'frequency': stat.frequency, 'interval': stat.interval}
+    data = {'end_times': end_times, 'frequency': stat.frequency}
     for table in tables:
         if table == RealmCount:
             data['realm'] = get_time_series_by_subgroup(
-                stat, RealmCount, realm.id, end_times, subgroups, labels, include_empty_subgroups)
+                stat, RealmCount, realm.id, end_times, subgroup_to_label, include_empty_subgroups)
         if table == UserCount:
             data['user'] = get_time_series_by_subgroup(
-                stat, UserCount, user_profile.id, end_times, subgroups, labels, include_empty_subgroups)
+                stat, UserCount, user_profile.id, end_times, subgroup_to_label, include_empty_subgroups)
+    if labels_sort_function is not None:
+        data['display_order'] = labels_sort_function(data)
+    else:
+        data['display_order'] = None
     return json_success(data=data)
 
+def sort_by_totals(value_arrays):
+    # type: (Dict[str, List[int]]) -> List[str]
+    totals = [(sum(values), label) for label, values in value_arrays.items()]
+    totals.sort(reverse=True)
+    return [label for total, label in totals]
+
+# For any given user, we want to show a fixed set of clients in the chart,
+# regardless of the time aggregation or whether we're looking at realm or
+# user data. This fixed set ideally includes the clients most important in
+# understanding the realm's traffic and the user's traffic. This function
+# tries to rank the clients so that taking the first N elements of the
+# sorted list has a reasonable chance of doing so.
+def sort_client_labels(data):
+    # type: (Dict[str, Dict[str, List[int]]]) -> List[str]
+    realm_order = sort_by_totals(data['realm'])
+    user_order = sort_by_totals(data['user'])
+    label_sort_values = {}  # type: Dict[str, float]
+    for i, label in enumerate(realm_order):
+        label_sort_values[label] = i
+    for i, label in enumerate(user_order):
+        label_sort_values[label] = min(i-.1, label_sort_values.get(label, i))
+    return [label for label, sort_value in sorted(label_sort_values.items(),
+                                                  key=lambda x: x[1])]
+
 def table_filtered_to_id(table, key_id):
     # type: (Type[BaseCount], int) -> QuerySet
     if table == RealmCount:
@@ -106,7 +150,7 @@ def table_filtered_to_id(table, key_id):
     elif table == InstallationCount:
         return InstallationCount.objects.all()
     else:
-        raise ValueError("Unknown table: %s" % (table,))
+        raise AssertionError("Unknown table: %s" % (table,))
 
 def client_label_map(name):
     # type: (str) -> str
@@ -128,7 +172,7 @@ def client_label_map(name):
 
 def rewrite_client_arrays(value_arrays):
     # type: (Dict[str, List[int]]) -> Dict[str, List[int]]
-    mapped_arrays = {} # type: Dict[str, List[int]]
+    mapped_arrays = {}  # type: Dict[str, List[int]]
     for label, array in value_arrays.items():
         mapped_label = client_label_map(label)
         if mapped_label in mapped_arrays:
@@ -138,20 +182,15 @@ def rewrite_client_arrays(value_arrays):
             mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
     return mapped_arrays
 
-def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroups, labels, include_empty_subgroups):
-    # type: (CountStat, Type[BaseCount], Optional[int], List[datetime], List[str], Optional[List[str]], bool) -> Dict[str, List[int]]
-    if labels is None:
-        labels = subgroups
-    if len(subgroups) != len(labels):
-        raise ValueError("subgroups and labels have lengths %s and %s, which are different." %
-                         (len(subgroups), len(labels)))
+def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroup_to_label, include_empty_subgroups):
+    # type: (CountStat, Type[BaseCount], int, List[datetime], Dict[Optional[str], str], bool) -> Dict[str, List[int]]
     queryset = table_filtered_to_id(table, key_id).filter(property=stat.property) \
                                                   .values_list('subgroup', 'end_time', 'value')
-    value_dicts = defaultdict(lambda: defaultdict(int)) # type: Dict[Optional[str], Dict[datetime, int]]
+    value_dicts = defaultdict(lambda: defaultdict(int))  # type: Dict[Optional[str], Dict[datetime, int]]
     for subgroup, end_time, value in queryset:
         value_dicts[subgroup][end_time] = value
     value_arrays = {}
-    for subgroup, label in zip(subgroups, labels):
+    for subgroup, label in subgroup_to_label.items():
         if (subgroup in value_dicts) or include_empty_subgroups:
             value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
 
@@ -222,7 +261,7 @@ def get_realm_day_counts():
     rows = dictfetchall(cursor)
     cursor.close()
 
-    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
+    counts = defaultdict(dict)  # type: Dict[str, Dict[int, int]]
     for row in rows:
         counts[row['string_id']][row['age']] = row['cnt']
 
@@ -702,12 +741,12 @@ def ad_hoc_queries():
 
     return pages
 
-@zulip_internal
+@require_server_admin
 @has_request_variables
 def get_activity(request):
     # type: (HttpRequest) -> HttpResponse
-    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
-    counts_content = realm_summary_table(realm_minutes) # type: str
+    duration_content, realm_minutes = user_activity_intervals()  # type: Tuple[mark_safe, Dict[str, float]]
+    counts_content = realm_summary_table(realm_minutes)  # type: str
     data = [
         ('Counts', counts_content),
         ('Durations', duration_content),
@@ -717,10 +756,10 @@ def get_activity(request):
 
     title = 'Activity'
 
-    return render_to_response(
+    return render(
+        request,
         'analytics/activity.html',
-        dict(data=data, title=title, is_home=True),
-        request=request
+        context=dict(data=data, title=title, is_home=True),
     )
 
 def get_user_activity_records_for_realm(realm, is_bot):
@@ -789,7 +828,7 @@ def get_user_activity_summary(records):
     #: We could use something like:
     # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
     #: but that would require this long `Union` to carry on throughout inner functions.
-    summary = {} # type: Dict[str, Dict[str, Any]]
+    summary = {}  # type: Dict[str, Dict[str, Any]]
 
     def update(action, record):
         # type: (str, QuerySet) -> None
@@ -952,7 +991,7 @@ def realm_user_summary_table(all_records, admin_emails):
 
     def is_recent(val):
         # type: (Optional[datetime]) -> bool
-        age = datetime.now(val.tzinfo) - val
+        age = timezone_now() - val
         return age.total_seconds() < 5 * 60
 
     rows = []
@@ -974,7 +1013,7 @@ def realm_user_summary_table(all_records, admin_emails):
         rows.append(row)
 
     def by_used_time(row):
-        # type: (Dict[str, Sequence[str]]) -> str
+        # type: (Dict[str, Any]) -> str
         return row['cells'][3]
 
     rows = sorted(rows, key=by_used_time, reverse=True)
@@ -996,11 +1035,11 @@ def realm_user_summary_table(all_records, admin_emails):
     content = make_table(title, cols, rows, has_row_class=True)
     return user_records, content
 
-@zulip_internal
+@require_server_admin
 def get_realm_activity(request, realm_str):
     # type: (HttpRequest, str) -> HttpResponse
-    data = [] # type: List[Tuple[str, str]]
-    all_user_records = {} # type: Dict[str, Any]
+    data = []  # type: List[Tuple[str, str]]
+    all_user_records = {}  # type: Dict[str, Any]
 
     try:
         admins = Realm.objects.get(string_id=realm_str).get_admin_users()
@@ -1029,18 +1068,18 @@ def get_realm_activity(request, realm_str):
     realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (realm_str,)
 
     title = realm_str
-    return render_to_response(
+    return render(
+        request,
         'analytics/activity.html',
-        dict(data=data, realm_link=realm_link, title=title),
-        request=request
+        context=dict(data=data, realm_link=realm_link, title=title),
     )
 
-@zulip_internal
+@require_server_admin
 def get_user_activity(request, email):
     # type: (HttpRequest, str) -> HttpResponse
     records = get_user_activity_records_for_email(email)
 
-    data = [] # type: List[Tuple[str, str]]
+    data = []  # type: List[Tuple[str, str]]
     user_summary = get_user_activity_summary(records)
     content = user_activity_summary_table(user_summary)
 
@@ -1050,8 +1089,8 @@ def get_user_activity(request, email):
     data += [('Info', content)]
 
     title = email
-    return render_to_response(
+    return render(
+        request,
         'analytics/activity.html',
-        dict(data=data, title=title),
-        request=request
+        context=dict(data=data, title=title),
     )

api/README.md

@@ -44,7 +44,12 @@ Alternatively, you may explicitly use "--user", "--api-key", and
 `--site` in our examples, which is especially useful when testing.  If
 you are running several bots which share a home directory, we
 recommend using `--config` to specify the path to the `zuliprc` file
-for a specific bot.
+for a specific bot.  Finally, you can control the defaults for all of
+these variables using the environment variables `ZULIP_CONFIG`,
+`ZULIP_API_KEY`, `ZULIP_EMAIL`, `ZULIP_SITE`, `ZULIP_CERT`,
+`ZULIP_CERT_KEY`, and `ZULIP_CERT_BUNDLE`.  Command-line options take
+precedence over environment variables take precedence over the config
+files.
 
 The command line equivalents for other configuration options are:
 

api/bots/commute/commute.py

@@ -6,13 +6,13 @@ from os.path import expanduser
 from six.moves import configparser as cp
 
 home = expanduser('~')
-CONFIG_PATH = home + '/google-commute.ini'
+CONFIG_PATH = home + '/commute.config'
 
 class CommuteHandler(object):
     '''
     This plugin provides information regarding commuting
     from an origin to a destination, providing a multitude of information.
-    It looks for messages starting with '@commute'.
+    It looks for messages starting with @mention of the bot.
     '''
 
     def __init__(self):
@@ -25,7 +25,7 @@ class CommuteHandler(object):
             It can vary outputs depending on traffic conditions, departure and
             arrival times as well as user preferences
             (toll avoidance, preference for bus travel, etc.).
-            It looks for messages starting with '@commute'.
+            It looks for messages starting with @mention of the bot.
 
             Users should input an origin and a destination
             in any stream or through private messages to the bot to receive a
@@ -33,8 +33,8 @@ class CommuteHandler(object):
             input was originally private.
 
             Sample input:
-            @commute origins=Chicago,IL,USA destinations=New+York,NY,USA
-            @commute help
+            @mention-botname origins=Chicago,IL,USA destinations=New+York,NY,USA
+            @mention-botname help
             '''
 
     help_info = '''
@@ -58,7 +58,7 @@ class CommuteHandler(object):
         e.g. language=fr
 
     Sample request:
-        @commute origins=Chicago,IL,USA destinations=New+York,NY,USA language=fr
+        @mention-botname origins=Chicago,IL,USA destinations=New+York,NY,USA language=fr
 
     Please note:
         Fare information can be derived, though is solely dependent on the
@@ -77,22 +77,12 @@ class CommuteHandler(object):
         you can use the timezone bot.
                 '''
 
-    def triage_message(self, message, client):
-        original_content = message['content']
-        # This next line of code is defensive, as we
-        # never want to get into an infinite loop of posting follow
-        # ups for own follow ups!
-        if message['display_recipient'] == 'commute':
-            return False
-        is_commute = original_content.startswith('@commute')
-        return is_commute
-
     # adds API Authentication Key to url request
     def get_api_key(self):
-        # google-commute.ini must have been moved from
-        # ~/zulip/contrib_bots/bots/commute_bot/CommuteBot/google-commute.ini into
-        # /google-commute.ini for program to work
-        # see doc.md for more information
+        # commute.config must be moved from
+        # ~/zulip/api/bots/commute/commute.config into
+        # ~/commute.config for program to work
+        # see readme.md for more information
         with open(CONFIG_PATH) as settings:
             config = cp.ConfigParser()
             config.readfp(settings)
@@ -100,19 +90,7 @@ class CommuteHandler(object):
 
     # determines if bot will respond as a private message/ stream message
     def send_info(self, message, letter, client):
-        if message['type'] == 'private':
-            client.send_message(dict(
-                type='private',
-                to=message['sender_email'],
-                content=letter,
-            ))
-        else:
-            client.send_message(dict(
-                type='stream',
-                subject=message['subject'],
-                to=message['display_recipient'],
-                content=letter,
-            ))
+        client.send_reply(message, letter)
 
     def calculate_seconds(self, time_str):
         times = time_str.split(',')
@@ -220,13 +198,13 @@ class CommuteHandler(object):
 
     def handle_message(self, message, client, state_handler):
         original_content = message['content']
-        content_list = original_content.split()
+        query = original_content.split()
 
-        if "help" in content_list:
+        if "help" in query:
             self.send_info(message, self.help_info, client)
             return
 
-        params = self.parse_pair(content_list)
+        params = self.parse_pair(query)
         params['key'] = self.api_key
         self.add_time_to_params(params)
 

api/bots/commute/readme.md

@@ -11,24 +11,24 @@ units and information in various languages.
 The bot will respond to the same stream input was in. And if called as
 private message, the bot will reply with a private message.
 
-To setup the bot, you will first need to move google-commute.ini into
+To setup the bot, you will first need to move commute.config into
 the user home directory and add an API key.
 
 Move
 
 ```
-~/zulip/contrib_bots/bots/commute_bot/CommuteBot/google-commute.ini
+~/zulip/api/bots/commute/commute.config
 ```
 
 into
 
 ```
-~/google-commute.ini
+~/commute.config
 ```
 
 To add an API key, please visit:
 https://developers.google.com/maps/documentation/distance-matrix/start
-to retrieve a key and copy your api key into google-commute.ini
+to retrieve a key and copy your api key into commute.config
 
 Sample input and output:
 

api/bots/converter/converter.py

@@ -0,0 +1,130 @@
+# See readme.md for instructions on running this code.
+
+from __future__ import absolute_import
+from __future__ import division
+
+import copy
+import importlib
+import sys
+from math import log10, floor
+
+import utils
+import re
+
+def is_float(value):
+    try:
+        float(value)
+        return True
+    except ValueError:
+        return False
+
+# Rounds the number 'x' to 'digits' significant digits.
+# A normal 'round()' would round the number to an absolute amount of
+# fractional decimals, e.g. 0.00045 would become 0.0.
+# 'round_to()' rounds only the digits that are not 0.
+# 0.00045 would then become 0.0005.
+def round_to(x, digits):
+    return round(x, digits-int(floor(log10(abs(x)))))
+
+class ConverterHandler(object):
+    '''
+    This plugin allows users to make conversions between various units,
+    e.g. Celsius to Fahrenheit, or kilobytes to gigabytes.
+    It looks for messages of the format
+    '@mention-bot <number> <unit_from> <unit_to>'
+    The message '@mention-bot help' posts a short description of how to use
+    the plugin, along with a list of all supported units.
+    '''
+
+    def usage(self):
+        return '''
+               This plugin allows users to make conversions between
+               various units, e.g. Celsius to Fahrenheit,
+               or kilobytes to gigabytes. It looks for messages of
+               the format '@mention-bot <number> <unit_from> <unit_to>'
+               The message '@mention-bot help' posts a short description of
+               how to use the plugin, along with a list of
+               all supported units.
+               '''
+
+    def handle_message(self, message, client, state_handler):
+        bot_response = get_bot_converter_response(message, client)
+        client.send_reply(message, bot_response)
+
+def get_bot_converter_response(message, client):
+    content = message['content']
+
+    words = content.lower().split()
+    convert_indexes = [i for i, word in enumerate(words) if word == "@convert"]
+    convert_indexes = [-1] + convert_indexes
+    results = []
+
+    for convert_index in convert_indexes:
+        if (convert_index + 1) < len(words) and words[convert_index + 1] == 'help':
+            results.append(utils.HELP_MESSAGE)
+            continue
+        if (convert_index + 3) < len(words):
+            number = words[convert_index + 1]
+            unit_from = utils.ALIASES.get(words[convert_index + 2], words[convert_index + 2])
+            unit_to = utils.ALIASES.get(words[convert_index + 3], words[convert_index + 3])
+            exponent = 0
+
+            if not is_float(number):
+                results.append(number + ' is not a valid number. ' + utils.QUICK_HELP)
+                continue
+
+            number = float(number)
+            number_res = copy.copy(number)
+
+            for key, exp in utils.PREFIXES.items():
+                if unit_from.startswith(key):
+                    exponent += exp
+                    unit_from = unit_from[len(key):]
+                if unit_to.startswith(key):
+                    exponent -= exp
+                    unit_to = unit_to[len(key):]
+
+            uf_to_std = utils.UNITS.get(unit_from, False)
+            ut_to_std = utils.UNITS.get(unit_to, False)
+
+            if uf_to_std is False:
+                results.append(unit_from + ' is not a valid unit. ' + utils.QUICK_HELP)
+            if ut_to_std is False:
+                results.append(unit_to + ' is not a valid unit.' + utils.QUICK_HELP)
+            if uf_to_std is False or ut_to_std is False:
+                continue
+
+            base_unit = uf_to_std[2]
+            if uf_to_std[2] != ut_to_std[2]:
+                unit_from = unit_from.capitalize() if uf_to_std[2] == 'kelvin' else unit_from
+                results.append(unit_to.capitalize() + ' and ' + unit_from +
+                               ' are not from the same category. ' + utils.QUICK_HELP)
+                continue
+
+            # perform the conversion between the units
+            number_res *= uf_to_std[1]
+            number_res += uf_to_std[0]
+            number_res -= ut_to_std[0]
+            number_res /= ut_to_std[1]
+
+            if base_unit == 'bit':
+                number_res *= 1024 ** (exponent // 3)
+            else:
+                number_res *= 10 ** exponent
+            number_res = round_to(number_res, 7)
+
+            results.append('{} {} = {} {}'.format(number,
+                                                  words[convert_index + 2],
+                                                  number_res,
+                                                  words[convert_index + 3]))
+
+        else:
+            results.append('Too few arguments given. ' + utils.QUICK_HELP)
+
+    new_content = ''
+    for idx, result in enumerate(results, 1):
+        new_content += ((str(idx) + '. conversion: ') if len(results) > 1 else '') + result + '\n'
+
+    return new_content
+
+handler_class = ConverterHandler

api/bots/converter/test_converter.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestConverterBot(BotTestCase):
+    bot_name = "converter"
+
+    def test_bot(self):
+        expected = {
+            "": ('Too few arguments given. Enter `@convert help` '
+                 'for help on using the converter.\n'),
+            "foo bar": ('Too few arguments given. Enter `@convert help` '
+                        'for help on using the converter.\n'),
+            "2 m cm": "2.0 m = 200.0 cm\n",
+            "12.0 celsius fahrenheit": "12.0 celsius = 53.600054 fahrenheit\n",
+            "0.002 kilometer millimile": "0.002 kilometer = 1.2427424 millimile\n",
+            "3 megabyte kilobit": "3.0 megabyte = 24576.0 kilobit\n",
+        }
+        self.check_expected_responses(expected)

api/bots/define/define.py

@@ -7,7 +7,7 @@ import html2text
 class DefineHandler(object):
     '''
     This plugin define a word that the user inputs. It
-    looks for messages starting with '@define'.
+    looks for messages starting with '@mention-bot'.
     '''
 
     DEFINITION_API_URL = 'https://owlbot.info/api/v1/dictionary/{}?format=json'
@@ -15,70 +15,52 @@ class DefineHandler(object):
     EMPTY_WORD_REQUEST_ERROR_MESSAGE = 'Please enter a word to define.'
     PHRASE_ERROR_MESSAGE = 'Definitions for phrases are not available.'
 
-    def usage(DefineHandler):
+    def usage(self):
         return '''
             This plugin will allow users to define a word. Users should preface
-            messages with "@define".
+            messages with @mention-bot.
             '''
 
-    def triage_message(DefineHandler, message, client):
-        original_content = message['content']
-        # This next line of code is defensive, as we
-        # never want to get into an infinite loop of posting follow
-        # ups for own follow ups!
-        is_define = original_content.startswith('@define')
+    def handle_message(self, message, client, state_handler):
+        original_content = message['content'].strip()
+        bot_response = self.get_bot_define_response(original_content)
 
-        return is_define
+        client.send_reply(message, bot_response)
 
-    def _handle_definition(DefineHandler, original_content):
-        # Remove '@define' from the message and extract the rest of the message, the
-        # word to define.
+    def get_bot_define_response(self, original_content):
         split_content = original_content.split(' ')
-
         # If there are more than one word (a phrase)
-        if len(split_content) > 2:
+        if len(split_content) > 1:
             return DefineHandler.PHRASE_ERROR_MESSAGE
 
-        to_define = split_content[1].strip()
+        to_define = split_content[0].strip()
         to_define_lower = to_define.lower()
 
         # No word was entered.
         if not to_define_lower:
-            return DefineHandler.EMPTY_WORD_REQUEST_ERROR_MESSAGE
+            return self.EMPTY_WORD_REQUEST_ERROR_MESSAGE
         else:
             response = '**{}**:\n'.format(to_define)
 
             try:
                 # Use OwlBot API to fetch definition.
-                api_result = requests.get(DefineHandler.DEFINITION_API_URL.format(to_define_lower))
+                api_result = requests.get(self.DEFINITION_API_URL.format(to_define_lower))
                 # Convert API result from string to JSON format.
                 definitions = api_result.json()
 
                 # Could not fetch definitions for the given word.
                 if not definitions:
-                    response += DefineHandler.REQUEST_ERROR_MESSAGE
-                else: # Definitions available.
+                    response += self.REQUEST_ERROR_MESSAGE
+                else:  # Definitions available.
                     # Show definitions line by line.
                     for d in definitions:
                         example = d['example'] if d['example'] else '*No example available.*'
                         response += '\n' + '* (**{}**) {}\n  {}'.format(d['type'], d['defenition'], html2text.html2text(example))
 
             except Exception as e:
-                response += DefineHandler.REQUEST_ERROR_MESSAGE
+                response += self.REQUEST_ERROR_MESSAGE
                 logging.exception(e)
 
             return response
 
-    def handle_message(DefineHandler, message, client, state_handler):
-        original_content = message['content']
-
-        response = DefineHandler._handle_definition(original_content)
-
-        client.send_message(dict(
-            type='stream',
-            to=message['display_recipient'],
-            subject=message['sender_email'],
-            content=response
-        ))
-
 handler_class = DefineHandler

api/bots/define/readme.md

@@ -12,10 +12,10 @@ is not in the dictionary, the definition is not displayed.
 * For example, if the user says "@define crash", all the meanings of crash
 appear, each in a separate line.
 
-![Correct Word](correct_word.png)
+![Correct Word](assets/correct_word.png)
 
 * If the user enters a wrong word, like "@define cresh" or "@define crish",
 then an error message saying no definition is available is displayed.
 
-![Wrong Word](wrong_word.png)
+![Wrong Word](assets/wrong_word.png)
 

api/bots/define/test_define.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestDefineBot(BotTestCase):
+    bot_name = "define"
+
+    def test_bot(self):
+        expected = {
+            "": 'Please enter a word to define.',
+            "foo": "**foo**:\nDefinition not available.",
+            "cat": ("**cat**:\n\n* (**noun**) a small domesticated carnivorous mammal "
+                    "with soft fur, a short snout, and retractile claws. It is widely "
+                    "kept as a pet or for catching mice, and many breeds have been "
+                    "developed.\n  their pet cat\n\n"),
+        }
+        self.check_expected_responses(expected)

api/bots/encrypt/encrypt.py

@@ -18,36 +18,25 @@ def encrypt(text):
 class EncryptHandler(object):
     '''
     This bot allows users to quickly encrypt messages using ROT13 encryption.
-    It encrypts/decrypts messages starting with @encrypt.
+    It encrypts/decrypts messages starting with @mention-bot.
     '''
 
     def usage(self):
         return '''
             This bot uses ROT13 encryption for its purposes.
-            It responds to me starting with @encrypt.
+            It responds to me starting with @mention-bot.
             Feeding encrypted messages into the bot decrypts them.
             '''
 
-    def triage_message(self, message, client):
-
-        original_content = message['content']
-
-        # This makes sure that the bot only replies to messages it supposed to reply to.
-        should_be_encrypted = original_content.startswith('@encrypt')
-
-        return should_be_encrypted
-
     def handle_message(self, message, client, state_handler):
-        original_content = message['content']
-        temp_content = encrypt(original_content.replace('@encrypt', ''))
-        send_content = "Encrypted/Decrypted text: " + temp_content
+        bot_response = self.get_bot_encrypt_response(message)
+        client.send_reply(message, bot_response)
 
-        client.send_message(dict(
-            type='stream',
-            to=message['display_recipient'],
-            subject=message['subject'],
-            content = send_content
-        ))
+    def get_bot_encrypt_response(self, message):
+        original_content = message['content']
+        temp_content = encrypt(original_content)
+        send_content = "Encrypted/Decrypted text: " + temp_content
+        return send_content
 
 handler_class = EncryptHandler
 

api/bots/encrypt/test_encrypt.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestEncryptBot(BotTestCase):
+    bot_name = "encrypt"
+
+    def test_bot(self):
+        expected = {
+            "": "Encrypted/Decrypted text: ",
+            "Let\'s Do It": "Encrypted/Decrypted text: Yrg\'f Qb Vg",
+            "me&mom together..!!": "Encrypted/Decrypted text: zr&zbz gbtrgure..!!",
+            "foo bar": "Encrypted/Decrypted text: sbb one",
+            "Please encrypt this": "Encrypted/Decrypted text: Cyrnfr rapelcg guvf",
+        }
+        self.check_expected_responses(expected)

api/bots/followup/followup.py

@@ -4,7 +4,7 @@ class FollowupHandler(object):
     '''
     This plugin facilitates creating follow-up tasks when
     you are using Zulip to conduct a virtual meeting.  It
-    looks for messages starting with '@followup'.
+    looks for messages starting with '@mention-bot'.
 
     In this example, we write follow up items to a special
     Zulip stream called "followup," but this code could
@@ -16,36 +16,31 @@ class FollowupHandler(object):
         return '''
             This plugin will allow users to flag messages
             as being follow-up items.  Users should preface
-            messages with "@followup".
+            messages with "@mention-bot".
 
             Before running this, make sure to create a stream
             called "followup" that your API user can send to.
             '''
 
-    def triage_message(self, message, client):
-        original_content = message['content']
-
-        # This next line of code is defensive, as we
-        # never want to get into an infinite loop of posting follow
-        # ups for own follow ups!
-        if message['display_recipient'] == 'followup':
-            return False
-        is_follow_up = (original_content.startswith('@followup') or
-                        original_content.startswith('@follow-up'))
-
-        return is_follow_up
-
     def handle_message(self, message, client, state_handler):
+        if message['content'] == '':
+            bot_response = "Please specify the message you want to send to followup stream after @mention-bot"
+            client.send_reply(message, bot_response)
+        else:
+            bot_response = self.get_bot_followup_response(message)
+            client.send_message(dict(
+                type='stream',
+                to='followup',
+                subject=message['sender_email'],
+                content=bot_response,
+            ))
+
+    def get_bot_followup_response(self, message):
         original_content = message['content']
         original_sender = message['sender_email']
-        new_content = original_content.replace('@followup',
-                                               'from %s:' % (original_sender,))
+        temp_content = 'from %s: ' % (original_sender,)
+        new_content = temp_content + original_content
 
-        client.send_message(dict(
-            type='stream',
-            to='followup',
-            subject=message['sender_email'],
-            content=new_content,
-        ))
+        return new_content
 
 handler_class = FollowupHandler

api/bots/followup/test_followup.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestFollowUpBot(BotTestCase):
+    bot_name = "followup"
+
+    def test_bot(self):
+        expected_send_reply = {
+            "": 'Please specify the message you want to send to followup stream after @mention-bot'
+        }
+        self.check_expected_responses(expected_send_reply, expected_method='send_reply')
+
+        expected_send_message = {
+            "foo": {
+                'type': 'stream',
+                'to': 'followup',
+                'subject': 'foo_sender@zulip.com',
+                'content': 'from foo_sender@zulip.com: foo',
+            },
+            "I have completed my task": {
+                'type': 'stream',
+                'to': 'followup',
+                'subject': 'foo_sender@zulip.com',
+                'content': 'from foo_sender@zulip.com: I have completed my task',
+            },
+        }
+        self.check_expected_responses(expected_send_message, expected_method='send_message')

api/bots/foursquare/foursquare.py

@@ -9,13 +9,13 @@ from six.moves import configparser as cp
 from six.moves import range
 
 home = expanduser('~')
-CONFIG_PATH = home + '/zulip/contrib_bots/bots/foursquare/FourSquareBot/settings.ini'
+CONFIG_PATH = home + '/zulip/api/bots/foursquare/foursquare.config'
 
 def get_api_key():
-    # settings.ini must have been moved from
-    # ~/zulip/contrib_bots/bots/foursquare/FourSquareBot/settings.ini into
-    # ~/settings.ini for program to work
-    # see doc.md for more information
+    # foursquare.config must have been moved from
+    # ~/zulip/api/bots/foursquare/foursquare.config into
+    # ~/foursquare.config for program to work
+    # see readme.md for more information
     with open(CONFIG_PATH) as settings:
         config = cp.ConfigParser()
         config.readfp(settings)
@@ -30,13 +30,13 @@ class FoursquareHandler(object):
             This plugin allows users to search for restaurants nearby an inputted
             location to a limit of 3 venues for every location. The name, address
             and description of the restaurant will be outputted.
-            It looks for messages starting with '@foursquare'.
+            It looks for messages starting with '@mention-bot'.
             If you need help, simply type:
-            @foursquare /help into the Compose Message box
+            @mention-bot /help into the Compose Message box
 
             Sample input:
-            @foursquare Chicago, IL
-            @foursquare help
+            @mention-bot Chicago, IL
+            @mention-bot help
             '''
 
     help_info = '''
@@ -45,18 +45,10 @@ cusine of a restaurant in that exact order.
 Please note the required use of quotes in the search location.
 
 Example Inputs:
-@foursquare 'Millenium Park' 8000 donuts
-@foursquare 'Melbourne, Australia' 40000 seafood
+@mention-bot 'Millenium Park' 8000 donuts
+@mention-bot 'Melbourne, Australia' 40000 seafood
                 '''
 
-    def triage_message(self, message, client):
-        callers = ['@FourSquare', '@Foursquare', '@foursquare', '@4square', '@4sq']
-        for call in callers:
-            if call in message['content']:
-                return True
-                break
-        return False
-
     def format_json(self, venues):
         def format_venue(venue):
             name = venue['name']
@@ -68,19 +60,7 @@ Example Inputs:
         return '\n'.join(format_venue(venue) for venue in venues)
 
     def send_info(self, message, letter, client):
-        if message['type'] == 'private':
-            client.send_message(dict(
-                type='private',
-                to=message['sender_email'],
-                content=letter,
-            ))
-        else:
-            client.send_message(dict(
-                type='stream',
-                subject=message['subject'],
-                to=message['display_recipient'],
-                content=letter,
-            ))
+        client.send_reply(message, letter)
 
     def handle_message(self, message, client, state_handler):
         words = message['content'].split()
@@ -98,7 +78,7 @@ Example Inputs:
             pass
 
         # Optional params for HTTP request.
-        if len(words) >= 2:
+        if len(words) >= 1:
             try:
                 params['radius'] = re.search('([0-9]){3,}', message['content']).group(0)
             except AttributeError:
@@ -115,7 +95,7 @@ Example Inputs:
             received_json = response.json()
         else:
             self.send_info(message,
-                           "Invalid Request\nIf stuck, try '@foursquare help'.",
+                           "Invalid Request\nIf stuck, try '@mention-bot help'.",
                            client)
             return
 
@@ -127,7 +107,7 @@ Example Inputs:
             return
 
         self.send_info(message,
-                       "Invalid Request\nIf stuck, try '@foursquare help'.",
+                       "Invalid Request\nIf stuck, try '@mention-bot help'.",
                        client)
         return
 

api/bots/foursquare/readme.md

@@ -28,5 +28,5 @@ Food nearby 'Chicago, IL' coming right up:
 * If the user enters a wrong word, like "@foursquare 80000 donuts" or "@foursquare",
 then an error message saying invalid input will be displayed.
 
-* To get the required API Key, visit: https://developer.foursquare.com/overview/auth
+* To get the required API key, visit: https://developer.foursquare.com/overview/auth
 for more information.

api/bots/giphy/giphy.py

@@ -22,45 +22,20 @@ class GiphyHandler(object):
     '''
     This plugin posts a GIF in response to the keywords provided by the user.
     Images are provided by Giphy, through the public API.
-    The bot looks for messages starting with "@giphy" or @mention of the bot
+    The bot looks for messages starting with @mention of the bot
     and responds with a message with the GIF based on provided keywords.
     It also responds to private messages.
     '''
     def usage(self):
         return '''
             This plugin allows users to post GIFs provided by Giphy.
-            Users should preface keywords with "@giphy" or the Giphy-bot @mention.
+            Users should preface keywords with the Giphy-bot @mention.
             The bot responds also to private messages.
             '''
 
-    def triage_message(self, message, client):
-        # To prevent infinite loop in private message, bot will detect
-        # if the sender name is the bot name it will return false.
-        if message['type'] == 'private':
-            return client.full_name != message['sender_full_name']
-
-        original_content = message['content']
-        is_giphy_called = (original_content.startswith('@giphy ') or
-                           message['is_mentioned'])
-
-        return is_giphy_called
-
     def handle_message(self, message, client, state_handler):
         bot_response = get_bot_giphy_response(message, client)
-
-        if message['type'] == 'private':
-            client.send_message(dict(
-                type='private',
-                to=message['sender_email'],
-                content=bot_response,
-            ))
-        else:
-            client.send_message(dict(
-                type='stream',
-                to=message['display_recipient'],
-                subject=message['subject'],
-                content=bot_response,
-            ))
+        client.send_reply(message, bot_response)
 
 
 class GiphyNoResultException(Exception):
@@ -99,23 +74,9 @@ def get_url_gif_giphy(keyword, api_key):
 
 
 def get_bot_giphy_response(message, client):
-    # Handle the message that called through mention.
-    if message['is_mentioned']:
-        bot_mention = r'^@(\*\*{0}\*\*\s|{0}\s)(?=.*)'.format(client.full_name)
-        start_with_mention = re.compile(bot_mention).match(message['content'])
-        if start_with_mention:
-            keyword = message['content'][len(start_with_mention.group()):]
-        else:
-            return 'Please mention me first, then type the keyword.'
-    # Handle the message that called through the specified keyword.
-    elif message['content'].startswith('@giphy '):
-        keyword = message['content'][len('@giphy '):]
-    # Handle the private message.
-    elif message['type'] == 'private':
-        keyword = message['content']
-
     # Each exception has a specific reply should "gif_url" return a number.
     # The bot will post the appropriate message for the error.
+    keyword = message['content']
     try:
         gif_url = get_url_gif_giphy(keyword, get_giphy_api_key_from_config())
     except requests.exceptions.ConnectionError:

api/bots/giphy/test_giphy.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+import json
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+from bots.giphy import giphy
+
+def get_http_response_json(gif_url):
+    response_json = {
+        'meta': {
+            'status': 200
+        },
+        'data': {
+            'images': {
+                'original': {
+                    'url': gif_url
+                }
+            }
+        }
+    }
+    return response_json
+
+def get_bot_response(gif_url):
+    return ('[Click to enlarge](%s)'
+            '[](/static/images/interactive-bot/giphy/powered-by-giphy.png)'
+            % (gif_url))
+
+def get_http_request(keyword):
+    return {
+        'api_url': giphy.GIPHY_TRANSLATE_API,
+        'params': {
+            's': keyword,
+            'api_key': giphy.get_giphy_api_key_from_config()
+        }
+    }
+
+class TestGiphyBot(BotTestCase):
+    bot_name = "giphy"
+
+    def test_bot(self):
+        # This message calls `send_reply` function of BotHandlerApi
+        keyword = "Hello"
+        gif_url = "https://media4.giphy.com/media/3o6ZtpxSZbQRRnwCKQ/giphy.gif"
+        expectations = {
+            keyword: get_bot_response(gif_url)
+        }
+        self.check_expected_responses(
+            expectations=expectations,
+            http_request=get_http_request(keyword),
+            http_response=get_http_response_json(gif_url)
+        )

api/bots/git_hub_comment/git_hub_comment.py

@@ -1,4 +1,4 @@
-# See readme-github-comment-bot.md for instructions on running this code.
+# See readme.md for instructions on running this code.
 from __future__ import absolute_import
 from __future__ import print_function
 from . import github
@@ -15,13 +15,13 @@ class InputError(IndexError):
 class GitHubHandler(object):
     '''
     This plugin allows you to comment on a GitHub issue, under a certain repository.
-    It looks for messages starting with '@comment' or '@gcomment'.
+    It looks for messages starting with '@mention-bot'.
     '''
 
     def usage(self):
         return '''
             This bot will allow users to comment on a GitHub issue.
-            Users should preface messages with '@comment' or '@gcomment'.
+            Users should preface messages with '@mention-bot'.
             You will need to have a GitHub account.
 
             Before running this, make sure to get a GitHub OAuth token.
@@ -39,25 +39,11 @@ class GitHubHandler(object):
             '<repository_owner>/<repository>/<issue_number>/<your_comment>'.
             '''
 
-    def triage_message(self, message, client):
-        original_content = message['content']
-
-        is_comment = (original_content.startswith('@comment') or
-                      original_content.startswith('@gcomment'))
-
-        return is_comment
-
     def handle_message(self, message, client, state_handler):
         original_content = message['content']
         original_sender = message['sender_email']
 
-        # this handles the message if its starts with @comment
-        if original_content.startswith('@comment'):
-            handle_input(client, original_content, original_sender)
-
-        # handle if message starts with @gcomment
-        elif original_content.startswith('@gcomment'):
-            handle_input(client, original_content, original_sender)
+        handle_input(client, original_content, original_sender)
 
 handler_class = GitHubHandler
 

api/bots/git_hub_comment/readme.md

@@ -26,8 +26,8 @@ github_token = <oauth_token>   (The personal access token for the GitHub bot)
 Here is an example of running the `git_hub_comment` bot from
 inside a Zulip repo:
 
-    `cd ~/zulip/contrib_bots`
-    `./run.py bots/git_hub_comment/git_hub_comment.py --config-file ~/.zuliprc-prod`
+    `cd ~/zulip/api`
+    `bots_api/run.py bots/git_hub_comment/git_hub_comment.py --config-file ~/.zuliprc-prod`
 
 Once the bot code starts running, you will see a
 message explaining how to use the bot, as well as

api/bots/github_issues/github_issues.py

@@ -13,7 +13,7 @@ import urllib.request
 class IssueHandler(object):
     '''
     This plugin facilitates sending issues to github, when
-    an item is prefixed with '@issue' or '@bug'
+    an item is prefixed with '@mention-bot'.
 
     It will also write items to the issues stream, as well
     as reporting it to github
@@ -30,7 +30,7 @@ class IssueHandler(object):
     def usage(self):
         return '''
             This plugin will allow users to flag messages
-            as being issues with Zulip by using te prefix '@issue'
+            as being issues with Zulip by using te prefix '@mention-bot'.
 
             Before running this, make sure to create a stream
             called "issues" that your API user can send to.
@@ -47,30 +47,20 @@ class IssueHandler(object):
             github_token = <oauth_token>   (The personal access token for the GitHub bot)
             '''
 
-    def triage_message(self, message, client):
-        original_content = message['content']
-        # This next line of code is defensive, as we
-        # never want to get into an infinite loop of posting follow
-        # ups for own follow ups!
-        if message['display_recipient'] == 'issue':
-            return False
-        is_issue = original_content.startswith('@issue')
-        return is_issue
-
     def handle_message(self, message, client, state_handler):
 
         original_content = message['content']
         original_sender = message['sender_email']
 
-        new_content = original_content.replace('@issue', 'by {}:'.format(original_sender,))
+        temp_content = 'by {}:'.format(original_sender,)
+        new_content = temp_content + original_content
         # gets the repo url
         url_new = self.URL.format(self.REPO_OWNER, self.REPO_NAME)
 
         # signs into github using the provided username and password
         session = github.auth()
 
-        # Gets rid of the @issue in the issue title
-        issue_title = message['content'].replace('@issue', '').strip()
+        issue_title = message['content'].strip()
         issue_content = ''
         new_issue_title = ''
         for part_of_title in issue_title.split():

api/bots/googlesearch/googlesearch.py

@@ -0,0 +1,93 @@
+# See readme.md for instructions on running this code.
+from __future__ import print_function
+import logging
+import http.client
+from six.moves.urllib.request import urlopen
+
+# Uses the Google search engine bindings
+#   pip install --upgrade google
+from google import search
+
+
+def get_google_result(search_keywords):
+    help_message = "To use this bot, start messages with @mentioned-bot, \
+                    followed by what you want to search for. If \
+                    found, Zulip will return the first search result \
+                    on Google.\
+                    \
+                    An example message that could be sent is:\
+                    '@mentioned-bot zulip' or \
+                    '@mentioned-bot how to create a chatbot'."
+    if search_keywords == 'help':
+        return help_message
+    elif search_keywords == '' or search_keywords is None:
+        return help_message
+    else:
+        try:
+            urls = search(search_keywords, stop=20)
+            urlopen('http://216.58.192.142', timeout=1)
+        except http.client.RemoteDisconnected as er:
+            logging.exception(er)
+            return 'Error: No internet connection. {}.'.format(er)
+        except Exception as e:
+            logging.exception(e)
+            return 'Error: Search failed. {}.'.format(e)
+
+        try:
+            url = next(urls)
+        except AttributeError as a_err:
+            # google.search query failed and urls is of object
+            # 'NoneType'
+            logging.exception(a_err)
+            return "Error: Google search failed with a NoneType result. {}.".format(a_err)
+        except TypeError as t_err:
+            # google.search query failed and returned None
+            # This technically should not happen but the prior
+            # error check assumed this behavior
+            logging.exception(t_err)
+            return "Error: Google search function failed. {}.".format(t_err)
+        except Exception as e:
+            logging.exception(e)
+            return 'Error: Search failed. {}.'.format(e)
+
+        return 'Success: {}'.format(url)
+
+
+class GoogleSearchHandler(object):
+    '''
+    This plugin allows users to enter a search
+    term in Zulip and get the top URL sent back
+    to the context (stream or private) in which
+    it was called. It looks for messages starting
+    with @mentioned-bot.
+    '''
+
+    def usage(self):
+        return '''
+            This plugin will allow users to search
+            for a given search term on Google from
+            Zulip. Use '@mentioned-bot help' to get
+            more information on the bot usage. Users
+            should preface messages with
+            @mentioned-bot.
+            '''
+
+    def handle_message(self, message, client, state_handler):
+        original_content = message['content']
+        result = get_google_result(original_content)
+        client.send_reply(message, result)
+
+handler_class = GoogleSearchHandler
+
+
+def test():
+    try:
+        urlopen('http://216.58.192.142', timeout=1)
+        print('Success')
+        return True
+    except http.client.RemoteDisconnected as e:
+        print('Error: {}'.format(e))
+        return False
+
+if __name__ == '__main__':
+    test()

api/bots/googlesearch/readme.md

@@ -0,0 +1,23 @@
+# Google Search bot
+
+This bot allows users to do Google search queries and have the bot
+respond with the first search result.  It is by default set to the
+highest safe-search setting.
+
+## Usage
+
+Run this bot as described
+[here](http://zulip.readthedocs.io/en/latest/bots-guide.html#how-to-deploy-a-bot).
+
+Use this bot with the following command
+
+`@mentioned-bot <search terms>`
+
+This will return the first link found by Google for `<search terms>`
+and print the resulting URL.
+
+If no `<search terms>` are entered, a help message is printed instead.
+
+If there was an error in the process of running the search (socket
+errors, Google search function failed, or general failures), an error
+message is returned.

api/bots/helloworld/helloworld.py

@@ -0,0 +1,18 @@
+# See readme.md for instructions on running this code.
+
+
+class HelloWorldHandler(object):
+    def usage(self):
+        return '''
+        This is a boilerplate bot that responds to a user query with
+        "beep boop", which is robot for "Hello World".
+
+        This bot can be used as a template for other, more
+        sophisticated, bots.
+        '''
+
+    def handle_message(self, message, client, state_handler):
+        content = 'beep boop'
+        client.send_reply(message, content)
+
+handler_class = HelloWorldHandler

api/bots/helloworld/readme.md

@@ -0,0 +1,4 @@
+Simple Zulip bot that will respond to any query with a "beep boop".
+
+The helloworld bot is a boilerplate bot that can be used as a
+template for more sophisticated/evolved Zulip bots.

api/bots/helloworld/test_helloworld.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+from six.moves import zip
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestHelloWorldBot(BotTestCase):
+    bot_name = "helloworld"
+
+    def test_bot(self):
+        txt = "beep boop"
+        messages = ["", "foo", "Hi, my name is abc"]
+        self.check_expected_responses(dict(list(zip(messages, len(messages)*[txt]))))

api/bots/help/help.py

@@ -0,0 +1,18 @@
+# See readme.md for instructions on running this code.
+
+class HelpHandler(object):
+    def usage(self):
+        return '''
+            This plugin will give info about Zulip to
+            any user that types a message saying "help".
+
+            This is example code; ideally, you would flesh
+            this out for more useful help pertaining to
+            your Zulip instance.
+            '''
+
+    def handle_message(self, message, client, state_handler):
+        help_content = "Info on Zulip can be found here:\nhttps://github.com/zulip/zulip"
+        client.send_reply(message, help_content)
+
+handler_class = HelpHandler

api/bots/help/test_help.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import sys
+from six.moves import zip
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
+# For dev setups, we can find the API in the repo itself.
+if os.path.exists(os.path.join(our_dir, '..')):
+    sys.path.insert(0, '..')
+from bots_test_lib import BotTestCase
+
+class TestHelpBot(BotTestCase):
+    bot_name = "help"
+
+    def test_bot(self):
+        txt = "Info on Zulip can be found here:\nhttps://github.com/zulip/zulip"
+        messages = ["", "help", "Hi, my name is abc"]
+        self.check_expected_responses(dict(list(zip(messages, len(messages)*[txt]))))

api/bots/howdoi/howdoi.py

@@ -12,6 +12,8 @@ Note:
       required by `howdoi`.
 """
 
+from __future__ import absolute_import
+
 import sys
 import logging
 from textwrap import fill
@@ -30,16 +32,16 @@ class HowdoiHandler(object):
     in your questions.
 
     There are two possible commands:
-    * @howdowe > This would return the answer to the same
+    * @mention-bot howdowe > This would return the answer to the same
                  stream that it was called from.
 
-    * @howdoi > The bot would send a private message to the
+    * @mention-bot howdoi > The bot would send a private message to the
                 user containing the answer.
 
     By default, howdoi only returns the coding section of the
     first search result if possible, to see the full answer
     from Stack Overflow, append a '!' to the commands.
-    (ie '@howdoi!', '@howdowe!')
+    (ie '@mention-bot howdoi!', '@mention-bot howdowe!')
     '''
 
     MAX_LINE_LENGTH = 85
@@ -50,26 +52,12 @@ class HowdoiHandler(object):
             answers from Stackoverflow. Users should preface
             their questions with one of the following:
 
-                * @howdowe > Answer to the same stream
-                * @howdoi > Answer via private message
+                * @mention-bot howdowe > Answer to the same stream
+                * @mention-bot howdoi > Answer via private message
 
-                * @howdowe! OR @howdoi! > Full answer from SO
+                * @mention-bot howdowe! OR @mention-bot howdoi! > Full answer from SO
             '''
 
-    def triage_message(self, message, client):
-        cmd_list = ['@howdowe', '@howdoi', '@howdowe!', '@howdoi!']
-        question = message['content']
-
-        # This next line of code is defensive, as we never want
-        # to get into an infinite loop of searching answers
-        # from Stackoverflow!
-        if message['sender_email'].startswith('howdoi'):
-            return False
-
-        is_howdoi = any([question.startswith(cmd) for cmd in cmd_list])
-
-        return is_howdoi
-
     def line_wrap(self, string, length):
         lines = string.split("\n")
 
@@ -94,36 +82,36 @@ class HowdoiHandler(object):
         return answer
 
     def handle_message(self, message, client, state_handler):
-        question = message['content']
+        question = message['content'].strip()
 
-        if question.startswith('@howdowe!'):
+        if question.startswith('howdowe!'):
             client.send_message(dict(
                 type='stream',
                 to=message['display_recipient'],
                 subject=message['subject'],
-                content=self.get_answer('@howdowe!', question)
+                content=self.get_answer('howdowe!', question)
             ))
 
-        elif question.startswith('@howdoi!'):
+        elif question.startswith('howdoi!'):
             client.send_message(dict(
                 type='private',
                 to=message['sender_email'],
-                content=self.get_answer('@howdoi!', question)
+                content=self.get_answer('howdoi!', question)
             ))
 
-        elif question.startswith('@howdowe'):
+        elif question.startswith('howdowe'):
             client.send_message(dict(
                 type='stream',
                 to=message['display_recipient'],
                 subject=message['subject'],
-                content=self.get_answer('@howdowe', question)
+                content=self.get_answer('howdowe', question)
             ))
 
-        elif question.startswith('@howdoi'):
+        elif question.startswith('howdoi'):
             client.send_message(dict(
                 type='private',
                 to=message['sender_email'],
-                content=self.get_answer('@howdoi', question)
+                content=self.get_answer('howdoi', question)
             ))
 
 

api/bots/howdoi/readme.md

@@ -23,24 +23,24 @@ answer will be formatted differently depending the chosen command.
 
 Question -> `@howdowe use supervisor in elixir`
 
-  ![howdowe question](question_howdowe.png)
+  ![howdowe question](assets/question_howdowe.png)
 
 Answer ->  Howdoi would try to **only** respond with the coding section
 of the answer.
 
-  ![howdowe answer](answer_howdowe.png)
+  ![howdowe answer](assets/answer_howdowe.png)
 
 #### Example 2
 
 Question -> `@howdoi! stack vs heap`
 
-  ![howdoi! question](question_howdoi_all.png)
+  ![howdoi! question](assets/question_howdoi_all.png)
 
 Answer -> Howdoi would return the **full** stackoverflow answer via
 **private message** to the original sender. The URL of the answer can be
 seen at the bottom of the message.
 
-  ![howdoi! answer](answer_howdoi_all.png)
+  ![howdoi! answer](assets/answer_howdoi_all.png)
 
 **Note:**
 

api/bots/incrementor/incrementor.py

@@ -0,0 +1,30 @@
+# See readme.md for instructions on running this code.
+
+
+class IncrementorHandler(object):
+
+    def __init__(self):
+        self.number = 0
+        self.message_id = None
+
+    def usage(self):
+        return '''
+        This is a boilerplate bot that makes use of the
+        update_message function. For the first @-mention, it initially
+        replies with one message containing a `1`. Every time the bot
+        is @-mentioned, this number will be incremented in the same message.
+        '''
+
+    def handle_message(self, message, client, state_handler):
+        self.number += 1
+        if self.message_id is None:
+            result = client.send_reply(message, str(self.number))
+            self.message_id = result['id']
+        else:
+            client.update_message(dict(
+                message_id=self.message_id,
+                content=str(self.number),
+            ))
+
+
+handler_class = IncrementorHandler

api/bots/incrementor/readme.md

@@ -0,0 +1,6 @@
+# Incrementor bot
+
+This is a boilerplate bot that makes use of the
+update_message function. For the first @-mention, it initially
+replies with one message containing a `1`. Every time the bot
+is @-mentioned, this number will be incremented in the same message.

api/bots/john/john.py

@@ -11,14 +11,14 @@ except ImportError:
     raise ImportError("""It looks like you are missing chatterbot.
                       Please: pip install chatterbot""")
 
-CONTRIB_BOTS_DIR = os.path.dirname(os.path.abspath(__file__))
-os.chdir(os.path.dirname(CONTRIB_BOTS_DIR))
-sys.path.insert(0, os.path.dirname(CONTRIB_BOTS_DIR))
+BOTS_DIR = os.path.dirname(os.path.abspath(__file__))
+os.chdir(os.path.dirname(BOTS_DIR))
+sys.path.insert(0, os.path.dirname(BOTS_DIR))
 
-JOKES_PATH = os.path.join(CONTRIB_BOTS_DIR, 'John/var/jokes.json')
-DATABASE_PATH = os.path.join(CONTRIB_BOTS_DIR, 'John/var/database.db')
-DIRECTORY_PATH = os.path.join(CONTRIB_BOTS_DIR, 'John')
-VAR_PATH = os.path.join(CONTRIB_BOTS_DIR, 'John/var')
+JOKES_PATH = os.path.join(BOTS_DIR, 'assets/var/jokes.json')
+DATABASE_PATH = os.path.join(BOTS_DIR, 'assets/var/database.db')
+DIRECTORY_PATH = os.path.join(BOTS_DIR, 'assets')
+VAR_PATH = os.path.join(BOTS_DIR, 'assets/var')
 
 if not os.path.exists(DIRECTORY_PATH):
     os.makedirs(DIRECTORY_PATH)
@@ -38,7 +38,7 @@ def create_chat_bot(no_learn):
                            "response_selection_method": "chatterbot.response_selection.get_random_response",
                            "statement_comparison_function": "chatterbot.comparisons.levenshtein_distance"
                        }],
-                   output_adapter="chatterbot.output.OutputFormatAdapter",
+                   output_adapter="chatterbot.output.OutputAdapter",
                    output_format='text',
                    database=DATABASE_PATH,
                    silence_performance_warning="True",
@@ -109,22 +109,15 @@ class JohnHandler(object):
 
     def usage(self):
         return '''
-            Before running this, make sure to create a stream
-            called "VirtualHelp" that your API user can send to.
+            This bot aims to be Zulip's virtual assistant. It
+            finds the best match from a certain input.
+            Also understands the English language and can
+            mantain a conversation, joke and give useful information.
             '''
 
-    def triage_message(self, message, client):
-        original_content = message['content'].lower()
-        return (original_content.startswith("@john") or
-                original_content.startswith("@**john**"))
-
     def handle_message(self, message, client, state_handler):
         original_content = message['content']
-        client.send_message(dict(
-            type='stream',
-            to='VirtualHelp',
-            subject="John",
-            content=bota.get_response(original_content)
-        ))
+        bot_response = str(bota.get_response(original_content))
+        client.send_reply(message, bot_response)
 
 handler_class = JohnHandler

api/bots/john/readme.md

@@ -10,7 +10,7 @@ interactive bot that uses machine learning heuristics to simulate a
 conversation with the user. He has a great sense of humor and
 is also powered by Open Source code!
 
-![Joke John](joke.png)
+![Joke John](assets/joke.png)
 
 How it works?
 John is initially trained with Corpus files, or large text files.
@@ -20,11 +20,11 @@ try to find the response that best matches the input according to the Levenshtei
 which is a string metric for measuring the difference between two sequences. If several
 responses have the same acurracy, he will choose one at random.
 
-![Meet John](greetings.png)
+![Meet John](assets/greetings.png)
 
 Can he learn by himself?
 John's engine allows him to learn from his conversations with people. However,
 without strict supervision bots that learn from people can do harm, so learning
 is currently restricted to his initial corpus.
 
-![Assist](assist.png)
+![Assist](assets/assist.png)

api/bots/readme.md

@@ -29,8 +29,8 @@ to messages in any of the following settings:
 Here is an example of running the "follow-up" bot from
 inside a Zulip repo (and in your remote instance):
 
-    cd ~/zulip/contrib_bots
-    ./run.py bots/followup/followup.py --config-file ~/.zuliprc-prod
+    cd ~/zulip/api
+    bots_api/run.py bots/followup/followup.py --config-file ~/.zuliprc-prod
 
 Once the bot code starts running, you will see a
 message explaining how to use the bot, as well as
@@ -101,8 +101,8 @@ document you are reading now).
 
 Each bot library simply needs to do the following:
 
-- Define a class that supports the methods `usage`,
-`triage_message`, and `handle_message`.
+- Define a class that supports the methods `usage`
+and `handle_message`.
 - Set `handler_class` to be the name of that class.
 
 (We make this a two-step process to reduce code repetition