Upgrade: revert change to default LOCAL_UPLOADS_DIR in prod settings.

The main purpose of the "var" convention is to make it easy to write stuff inside of our git repo when running a dev instance, and then "var" gets excluded from checkins. For production, that's not as much of a concern. For upgrades we don't want to be changing the directory around and confusing matters, especially with the extra moving part of nginx configs (which have their own issues in terms of being overwritten by accident when admins go to S3).
upgrade: Create prod_settings symlink in step 2 if it doesn't exist.
2025-10-24 08:33:43 +00:00 · 2016-09-02 12:30:29 -07:00 · 2016-09-01 21:17:11 -07:00
2814 changed files with 116411 additions and 214801 deletions
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,19 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.{sh,py,js,json,yml,xml,css,md,markdown,handlebars,html}]
-indent_style = space
-indent_size = 4
-
-[*.{svg,rb,pp,pl}]
-indent_style = space
-indent_size = 2
-
-[*.{cfg}]
-indent_style = space
-indent_size = 8
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,3 +0,0 @@
-static/js/blueslip.js
-puppet/zulip_ops/files/statsd/local.js
-static/webpack-bundles
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,302 +0,0 @@
-{
-    "env": {
-        "node": true
-    },
-    "globals": {
-        "$": false,
-        "_": false,
-        "jQuery": false,
-        "Spinner": false,
-        "Handlebars": false,
-        "XDate": false,
-        "zxcvbn": false,
-        "LazyLoad": false,
-        "Dropbox": false,
-        "SockJS": false,
-        "marked": false,
-        "moment": false,
-        "i18n": false,
-        "DynamicText": false,
-        "bridge": false,
-        "page_params": false,
-        "status_classes": false,
-        "password_quality": false,
-        "attachments_ui": false,
-        "csrf_token": false,
-        "typeahead_helper": false,
-        "pygments_data": false,
-        "popovers": false,
-        "server_events": false,
-        "server_events_dispatch": false,
-        "ui": false,
-        "ui_report": false,
-        "ui_util": false,
-        "lightbox": false,
-        "stream_color": false,
-        "people": false,
-        "navigate": false,
-        "settings_account": false,
-        "settings_display": false,
-        "settings_notifications": false,
-        "settings_muting": false,
-        "settings_lab": false,
-        "settings_bots": false,
-        "settings_sections": false,
-        "settings_emoji": false,
-        "settings_org": false,
-        "settings_users": false,
-        "settings_streams": false,
-        "settings_filters": false,
-        "settings": false,
-        "resize": false,
-        "loading": false,
-        "typing": false,
-        "typing_events": false,
-        "typing_data": false,
-        "typing_status": false,
-        "compose": false,
-        "compose_actions": false,
-        "compose_state": false,
-        "compose_fade": false,
-        "overlays": false,
-        "stream_create": false,
-        "stream_edit": false,
-        "subs": false,
-        "stream_muting": false,
-        "stream_events": false,
-        "timerender": false,
-        "message_live_update": false,
-        "message_edit": false,
-        "reload": false,
-        "composebox_typeahead": false,
-        "search": false,
-        "topic_list": false,
-        "topic_generator": false,
-        "gear_menu": false,
-        "hashchange": false,
-        "hash_util": false,
-        "message_list": false,
-        "Filter": false,
-        "pointer": false,
-        "util": false,
-        "MessageListView": false,
-        "blueslip": false,
-        "rows": false,
-        "WinChan": false,
-        "muting_ui": false,
-        "Socket": false,
-        "channel": false,
-        "components": false,
-        "message_viewport": false,
-        "upload_widget": false,
-        "avatar": false,
-        "realm_icon": false,
-        "feature_flags": false,
-        "search_suggestion": false,
-        "referral": false,
-        "notifications": false,
-        "message_flags": false,
-        "bot_data": false,
-        "stream_sort": false,
-        "stream_list": false,
-        "stream_popover": false,
-        "narrow_state": false,
-        "narrow": false,
-        "narrow_state": false,
-        "admin_sections": false,
-        "admin": false,
-        "stream_data": false,
-        "list_util": false,
-        "muting": false,
-        "Dict": false,
-        "unread": false,
-        "alert_words_ui": false,
-        "message_store": false,
-        "message_util": false,
-        "message_events": false,
-        "message_fetch": false,
-        "favicon": false,
-        "condense": false,
-        "list_render": false,
-        "floating_recipient_bar": false,
-        "tab_bar": false,
-        "emoji": false,
-        "presence": false,
-        "activity": false,
-        "invite": false,
-        "colorspace": false,
-        "reactions": false,
-        "tutorial": false,
-        "templates": false,
-        "alert_words": false,
-        "fenced_code": false,
-        "markdown": false,
-        "echo": false,
-        "localstorage": false,
-        "localStorage": false,
-        "current_msg_list": true,
-        "home_msg_list": false,
-        "pm_list": false,
-        "pm_conversations": false,
-        "recent_senders": false,
-        "unread_ui": false,
-        "unread_ops": false,
-        "user_events": false,
-        "Plotly": false,
-        "emoji_codes": false,
-        "drafts": false,
-        "katex": false,
-        "Clipboard": false,
-        "emoji_picker": false,
-        "hotspots": false,
-        "compose_ui": false
-    },
-    "rules": {
-        "no-restricted-syntax": 0,
-        "no-nested-ternary": 0,
-        "spaced-comment": 0,
-        "space-infix-ops": 0,
-        "newline-per-chained-call": 0,
-        "no-whitespace-before-property": 0,
-        "padded-blocks": 0,
-        "space-in-parens": 0,
-        "eol-last": ["error", "always"],
-        "no-unneeded-ternary": ["error", { "defaultAssignment": false }],
-        "no-case-declarations": "error",
-        "eqeqeq": ["error", "allow-null"],
-        "no-duplicate-imports": "error",
-        "no-undef": "error",
-        "dot-notation": ["error", { "allowKeywords": true }],
-        "no-iterator": "error",
-        "no-dupe-class-members": "error",
-        "no-useless-constructor": "error",
-        "prefer-const": ["error", {
-            "destructuring": "any",
-            "ignoreReadBeforeAssign": true
-        }],
-        "no-const-assign": "error",
-        "no-new-object": 2,
-        "quote-props": ["error", "as-needed", {
-            "keywords": false,
-            "unnecessary": true,
-            "numbers": false
-        }],
-        "no-array-constructor": "error",
-        "array-callback-return": "error",
-        "template-curly-spacing": "error",
-        //The Zulip codebase complies partially with the "no-useless-escape" rule; only regex expressions haven't been updated yet.
-        //Updated regex expressions are currently being tested in casper files and will decide about a potential future enforcement of this rule.
-        "no-useless-escape": 0,
-        "func-style": ["off", "expression"],
-        "wrap-iife": ["error", "outside", { "functionPrototypeMethods": false }],
-        "no-new-func": "error",
-        "space-before-function-paren": ["error", { "anonymous": "always", "named": "never", "asyncArrow": "always" }],
-        "no-param-reassign": 0,
-        "arrow-spacing": ["error", { "before": true, "after": true }],
-        "no-alert": 2,
-        "no-array-constructor": 2,
-        "no-caller": 2,
-        "no-bitwise": 2,
-        "no-catch-shadow": 2,
-        "comma-dangle": ["error", {
-            "arrays": "always-multiline",
-            "objects": "always-multiline",
-            "imports": "always-multiline",
-            "exports": "always-multiline",
-            "functions": "never"
-        }],
-        "no-console": 0,
-        "no-control-regex": 2,
-        "no-debugger": 2,
-        "no-div-regex": 2,
-        "no-dupe-keys": 2,
-        "no-else-return": 2,
-        "no-empty": 2,
-        "no-empty-character-class": 2,
-        "no-eq-null": 2,
-        "no-eval": 2,
-        "no-ex-assign": 2,
-        "no-extra-semi": 2,
-        "no-func-assign": 2,
-        "no-floating-decimal": 2,
-        "no-implied-eval": 2,
-        "no-with": 2,
-        "no-fallthrough": 2,
-        "no-unreachable": 2,
-        "no-undef": 2,
-        "no-undef-init": 2,
-        "no-unused-expressions": 2,
-        "no-octal": 2,
-        "no-octal-escape": 2,
-        "no-obj-calls": 2,
-        "no-multi-str": 2,
-        "no-new-wrappers": 2,
-        "no-new": 2,
-        "no-new-func": 2,
-        "no-native-reassign": 2,
-        "no-plusplus": 2,
-        "no-delete-var": 2,
-        "no-return-assign": 2,
-        "no-new-object": 2,
-        "no-label-var": 2,
-        "no-ternary": 0,
-        "no-self-compare": 2,
-        "no-sync": 2,
-        "no-underscore-dangle": 0,
-        "no-loop-func": 2,
-        "no-labels": 2,
-        "no-unused-vars": ["error", { "vars": "local", "args": "after-used",
-                                      "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
-                                    }],
-        "no-script-url": 2,
-        "no-proto": 2,
-        "no-iterator": 2,
-        "no-mixed-requires": [0, false],
-        "no-extra-parens": ["error", "functions"],
-        "no-shadow": 0,
-        "no-use-before-define": 2,
-        "no-redeclare": 2,
-        "no-regex-spaces": 2,
-        "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
-        "block-scoped-var": 2,
-        "camelcase": 0,
-        "complexity": [0, 4],
-        "curly": 2,
-        "dot-notation": 2,
-        "guard-for-in": 2,
-        "max-depth": [0, 4],
-        "max-len": ["error", 100, 2, {
-            "ignoreUrls": true,
-            "ignoreComments": false,
-            "ignoreRegExpLiterals": true,
-            "ignoreStrings": true,
-            "ignoreTemplateLiterals": true
-        }],
-        "max-params": [0, 3],
-        "max-statements": [0, 10],
-        "new-cap": ["error", { "newIsCap": true, "capIsNew": false }],
-        "new-parens": 2,
-        "one-var": ["error", "never"],
-        "quotes": [0, "single"],
-        "quote-props": ["error", "as-needed", { "keywords": false, "unnecessary": true, "numbers": false }],
-        "radix": 2,
-        "semi": 2,
-        "keyword-spacing": ["error", {
-            "before": true,
-            "after": true,
-            "overrides": {
-                "return": { "after": true },
-                "throw": { "after": true },
-                "case": { "after": true }
-            }
-        }],
-        "space-before-blocks": 2,
-        "strict": 0,
-        "unnecessary-strict": 0,
-        "use-isnan": 2,
-        "valid-typeof": ["error", { "requireStringLiterals": true }],
-        "wrap-iife": 2,
-        "wrap-regex": 0,
-        "yoda": 2
-    }
-}
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,21 +1,13 @@
-*   text=auto eol=lf
-*.gif binary
-*.jpg binary
-*.eot binary
-*.woff binary
-*.svg binary
-*.ttf binary
-*.png binary
-*.otf binary
-*.tif binary
 .gitignore export-ignore
 .gitattributes export-ignore
-/static/assets export-ignore
+/analytics export-ignore
+/assets export-ignore
 /bots export-ignore
 /corporate export-ignore
 /static export-ignore
 /tools export-ignore
 /zilencer export-ignore
+/templates/analytics export-ignore
 /templates/corporate export-ignore
 /templates/zilencer export-ignore
 /puppet/zulip_internal export-ignore
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,10 @@
 *.pyc
 *~
 /prod-static
+/errors/*
 *.sw[po]
 *.DS_Store
+stats/
 .kdev4
 .idea
 zulip.kdev4
@@ -12,23 +14,13 @@ coverage/
 .kateproject.d/
 .kateproject
 *.kate-swp
-*.sublime-project
-*.sublime-workspace
 .vagrant
 /zproject/dev-secrets.conf
 static/js/bundle.js
-static/generated/emoji
-static/generated/pygments_data.js
-static/generated/github-contributors.json
+static/third/gemoji/
+static/third/zxcvbn/
 static/locale/language_options.json
-static/third/emoji-data
-static/webpack-bundles
-/node_modules
-/staticfiles.json
+node_modules
 npm-debug.log
 *.mo
 var/*
-.vscode/
-tools/conf.ini
-tools/custom_provision
-api/bots/john/assets/var/database.db
--- a/.gitlint
+++ b/.gitlint
@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing
-
-extra-path=tools/lib/gitlint-rules.py
-
-[title-match-regex]
-regex=^.+\.$
-
-[title-max-length]
-line-length=76
-
-[body-max-line-length]
-line-length=76
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,76 +1,47 @@
-# See https://zulip.readthedocs.io/en/latest/events-system.html for
-# high-level documentation on our Travis CI setup.
 dist: trusty
+before_install:
+   - nvm install 0.10
 install:
-  # Disable Travis CI's built-in NVM installation
-  - mv ~/.nvm ~/.travis-nvm-disabled
-
-  # Install coveralls, the library for the code coverage reporting tool we use
  - pip install coveralls
-
-  # This is the main setup job for the test suite
  - tools/travis/setup-$TEST_SUITE
-
-  # Clean any virtualenvs that are not in use to avoid our cache
-  # becoming huge.  TODO: Add similar cleanup code for the other caches.
  - tools/clean-venv-cache --travis
-script:
-  # We unset GEM_PATH here as a hack to work around Travis CI having
-  # broken running their system puppet with Ruby.  See
-  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
-  - unset GEM_PATH
-  - ./tools/travis/$TEST_SUITE
 cache:
  - apt: false
  - directories:
+    - $HOME/phantomjs
    - $HOME/zulip-venv-cache
-    - $HOME/zulip-npm-cache
-    - $HOME/zulip-emoji-cache
+    - node_modules
    - $HOME/node
 env:
  global:
-    - COVERAGE_FILE=var/.coverage
    - COVERALLS_PARALLEL=true
    - COVERALLS_SERVICE_NAME=travis-pro
    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
    - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
 language: python
-# We run all of our test suites for both Python 2.7 and 3.4, with the
-# exception of static analysis, which is just run once (and checks
-# against both Python versions).
+python:
+  - "2.7"
+  - "3.4"
 matrix:
  include:
    - python: "3.4"
      env: TEST_SUITE=static-analysis
-    - python: "3.4"
-      env: TEST_SUITE=production
    - python: "2.7"
      env: TEST_SUITE=production
-    - python: "2.7"
-      env: TEST_SUITE=frontend
-    - python: "3.4"
-      env: TEST_SUITE=frontend
-    - python: "2.7"
-      env: TEST_SUITE=backend
-    - python: "3.4"
-      env: TEST_SUITE=backend
+      sudo: required
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
 sudo: required
 services:
 - docker
 addons:
-  artifacts:
-    paths:
-      # Casper debugging data (screenshots, etc.) is super useful for
-      # debugging test flakes.
-      - $(ls var/casper/* | tr "\n" ":")
-      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
  postgresql: "9.3"
 after_success:
  coveralls
 notifications:
-  webhooks:
-    urls:
-      - https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
-      - https://zulip.org/zulipbot/travis
-    on_success: always
-    on_failure: always
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
--- a/.tx/config
+++ b/.tx/config
@@ -6,9 +6,12 @@ source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 file_filter = static/locale/<lang>/LC_MESSAGES/django.po
+lang_map = zh-Hans: zh_CN

 [zulip.translationsjson]
 source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
+lang_map = zh-Hans: zh-CN
+
--- a/4
+++ b/4
@@ -6,14 +6,10 @@ RUN apt-get update && apt-get install -y \
  python-pbs \
  wget

-RUN locale-gen en_US.UTF-8
-
 RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

 USER zulip

 RUN ln -nsf /srv/zulip ~/zulip

-RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
-
 WORKDIR /srv/zulip
--- a/README.md
+++ b/README.md
@@ -17,43 +17,31 @@ previews, group private messages, audible notifications,
 missed-message emails, desktop apps, and much more.

 Further information on the Zulip project and its features can be found
-at <https://www.zulip.org>.
+at https://www.zulip.org.

-[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master) [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/) [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master)

 ## Community

 There are several places online where folks discuss Zulip.

-* The primary place is the
-  [Zulip development community Zulip server](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html).
+One of those places is our [public Zulip instance](https://zulip.tabbott.net/).
+You can go through the simple signup process at that link, and then you
+will soon be talking to core Zulip developers and other users.  To get
+help in real time, you will have the best luck finding core developers
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get answered
+within a day.

-* For Google Summer of Code students and applicants, we have
-[a mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc)
-for help, questions, and announcements.  But it's often simpler to
-visit chat.zulip.org instead.
+We have a [Google mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
+that is currently pretty low traffic.  It is where we do things like
+announce public meetings or major releases.  You can also use it to
+ask questions about features or possible bugs.

-* We have
-[a public mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
-that is currently pretty low traffic because most discussions happen
-in our public Zulip instance.  We use it to announce Zulip developer
-community gatherings and ask for feedback on major technical or design
-decisions.  It has several hundred subscribers, so you can use it to
-ask questions about features or possible bugs, but please don't use it
-ask for generic help getting started as a contributor (e.g. because
-you want to do Google Summer of Code).  The rest of this page covers
-how to get involved in the Zulip project in detail.
-
-* Zulip also has a [blog](https://blog.zulip.org/) and
-  [twitter account](https://twitter.com/zuliposs).
-
-* Last but not least, we use [GitHub](https://github.com/zulip/zulip)
-to track Zulip-related issues (and store our code, of course).
-Anybody with a GitHub account should be able to create Issues there
-pertaining to bugs or enhancement requests.  We also use Pull Requests
-as our primary mechanism to receive code contributions.
-
-The Zulip community has a [Code of Conduct][code-of-conduct].
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
+track Zulip-related issues (and store our code, of course).
+Anybody with a Github account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull
+Requests as our primary mechanism to receive code contributions.

 ## Installing the Zulip Development environment

@@ -63,24 +51,20 @@ installation guide][dev-install].

 ## Running Zulip in production

-Zulip in production supports Ubuntu 14.04 Trusty and Ubuntu 16.04
-Xenial. Work is ongoing on adding support for additional
-platforms. The installation process is documented at
-<https://zulip.org/server.html> and in more detail in [the
+Zulip in production only supports Ubuntu 14.04 right now, but work is
+ongoing on adding support for additional platforms. The installation
+process is documented at https://zulip.org/server.html and in more
+detail in [the
 documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).

 ## Ways to contribute

-Zulip welcomes all forms of contributions!  This page documents the
+Zulip welcomes all forms of contributions!  The page documents the
 Zulip development process.

 * **Pull requests**. Before a pull request can be merged, you need to
-sign the [Dropbox Contributor License Agreement][cla].  Also,
+to sign the [Dropbox Contributor License Agreement][cla].  Also,
 please skim our [commit message style guidelines][doc-commit-style].
-We encourage early pull requests for work in progress. Prefix the title
-of your pull request with `[WIP]` and reference it when asking for
-community feedback. When you are ready for final review, remove
-the `[WIP]`.

 * **Testing**. The Zulip automated tests all run automatically when
 you submit a pull request, but you can also run them all in your
@@ -96,7 +80,7 @@ and [new feature tutorial][doc-newfeat]. You can also improve
 [Zulip.org][z-org].

 * **Mailing lists and bug tracker**. Zulip has a [development
-discussion mailing list](#community) and uses [GitHub issues
+discussion mailing list][gg-devel] and uses [GitHub issues
 ][gh-issues].  There are also lists for the [Android][email-android]
 and [iOS][email-ios] apps.  Feel free to send any questions or
 suggestions of areas where you'd love to see more documentation to the
@@ -104,49 +88,40 @@ relevant list!  Please report any security issues you discover to
 zulip-security@googlegroups.com.

 * **App codebases**. This repository is for the Zulip server and web
-app (including most integrations); the
-[React Native Mobile iOS app][ios-exp], [Android app][Android],
-[new Electron desktop app][electron], and
-[legacy QT-based desktop app][desktop] are all separate repositories.
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].

 * **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
 integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
-and [Trello][]), plus [node.js API bindings][node], an [isomorphic
- JavaScript library][zulip-js], and a [full-text search PostgreSQL
- extension][tsearch], as separate repos.
+and [Trello][]), plus [node.js API bindings][node], and a [full-text search
+PostgreSQL extension][tsearch], as separate repos.

 * **Translations**.  Zulip is in the process of being translated into
 10+ languages, and we love contributions to our translations.  See our
 [translating documentation][transifex] if you're interested in
 contributing!

-* **Code Reviews**. Zulip is all about community and helping each
-other out.  Check out [#code review][code-review] on
-[chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html)
-to help review PRs and give comments on other people's work. Everyone is
-welcome to participate, even those new to Zulip! Even just checking out
-the code, manually testing it, and posting on whether or not it worked
-is valuable.
-
 [cla]: https://opensource.dropbox.com/cla/
-[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
 [dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
 [doc]: https://zulip.readthedocs.io/
-[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/code-style.html#commit-messages
 [doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
 [doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
 [doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
 [electron]: https://github.com/zulip/zulip-electron
+[gg-devel]: https://groups.google.com/forum/#!forum/zulip-devel
 [gh-issues]: https://github.com/zulip/zulip/issues
 [desktop]: https://github.com/zulip/zulip-desktop
 [android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
 [ios-exp]: https://github.com/zulip/zulip-mobile
 [email-android]: https://groups.google.com/forum/#!forum/zulip-android
 [email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
 [hubot-adapter]: https://github.com/zulip/hubot-zulip
 [jenkins]: https://github.com/zulip/zulip-jenkins-plugin
 [node]: https://github.com/zulip/zulip-node
-[zulip-js]: https://github.com/zulip/zulip-js
 [phab]: https://github.com/zulip/phabricator-to-zulip
 [puppet]: https://github.com/matthewbarr/puppet-zulip
 [redmine]: https://github.com/zulip/zulip-redmine-plugin
@@ -154,31 +129,11 @@ is valuable.
 [tsearch]: https://github.com/zulip/tsearch_extras
 [transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
 [z-org]: https://github.com/zulip/zulip.github.io
-[code-review]: https://chat.zulip.org/#narrow/stream/code.20review
-
-## Google Summer of Code
-
-We participated in
-[GSoC](https://developers.google.com/open-source/gsoc/) in 2016 (with
-[great results](https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/))
-and are participating in 2017 as well.  For guidance, please read
-[our GSoC instructions and ideas page](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
-and feel free to email
-[our GSoC mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc).
-
-**Note**: For GSoC 2017, we will be focused on making our
-[React Native app](https://github.com/zulip/zulip-mobile) better
-rather than developing the
-[Java Android app](https://github.com/zulip/zulip-android) and
-[React Native app](https://github.com/zulip/zulip-mobile) in
-parallel. You can review
-[our detailed plan](https://github.com/zulip/zulip-android/blob/master/android-strategy.md)
-for further details on the motivation and logistics.

 ## How to get involved with contributing to Zulip

 First, subscribe to the Zulip [development discussion mailing
-list](#community).
+list][gg-devel].

 The Zulip project uses a system of labels in our [issue
 tracker][gh-issues] to make it easy to find a project if you don't
@@ -186,7 +141,7 @@ have your own project idea in mind or want to get some experience with
 working on Zulip before embarking on a larger project you have in
 mind:

-* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+* [Integrations](https://github.com/zulip/zulip/labels/integrations).
  Integrate Zulip with another piece of software and contribute it
  back to the community!  Writing an integration can be a great first
  contribution.  There's detailed documentation on how to write
@@ -196,7 +151,7 @@ mind:
 * [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
  Smaller projects that might be a great first contribution.

-* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+* [Documentation](https://github.com/zulip/zulip/labels/documentation):
  The Zulip project loves contributions of new documentation.

 * [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
@@ -212,35 +167,14 @@ mind:
  Browsing this list can be a great way to find feature ideas to
  implement that other Zulip users are excited about.

-* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
-  The projects that are
-  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
-  These are great projects if you're looking to make an impact.
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html): The
+  projects that are [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).  These are great projects if you're looking to make an impact.

-Another way to find issues in Zulip is to take advantage of our
-"area:<foo>" convention in separating out issues.  We partition all of
-our issues into areas like admin, compose, emoji, hotkeys, i18n,
-onboarding, search, etc.  You can see this here:
-
-<https://github.com/zulip/zulip/labels>
-
-Click on any of the "area:" labels and you will see all the tickets
-related to your area of interest.
-
-If you're excited about helping with an open issue, make sure to claim
-the issue by commenting the following in the comment section:
-"**@zulipbot** claim". **@zulipbot** will assign you to the issue and
-label the issue as **in progress**. For more details, check out
-[**@zulipbot**](https://github.com/zulip/zulipbot).
-
-You're encouraged to ask questions on how to best implement or debug
-your changes -- the Zulip maintainers are excited to answer questions
-to help you stay unblocked and working efficiently. It's great to ask
-questions in comments on GitHub issues and pull requests, or [on
-chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html). We'll
-direct longer discussions to Zulip chat, but please post a summary of
-what you learned from the chat, or link to the conversation, in a
-comment on the GitHub issue.
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.

 We also welcome suggestions of features that you feel would be
 valuable or changes that you feel would make Zulip a better open
@@ -270,16 +204,9 @@ Feedback on how to make this development process more efficient, fun,
 and friendly to new contributors is very welcome!  Just send an email
 to the Zulip Developers list with your thoughts.

-When you feel like you have completed your work on an issue, post your
-PR to the
-[#code review](https://chat.zulip.org/#narrow/stream/code.20review)
-stream on [chat.zulip.org](https://zulip.readthedocs.io/en/latest/chat-zulip-org.html).
-This is our lightweight process that gives other developers the
-opportunity to give you comments and suggestions on your work.
-
 ## License

-Copyright 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright 2011-2015 Dropbox, Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -296,4 +223,4 @@ limitations under the License.
 The software includes some works released by third parties under other
 free and open source licenses. Those works are redistributed under the
 license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.
+see the ``THIRDPARTY`` file included with this distribution.
--- a/docs/THIRDPARTY
+++ b/docs/THIRDPARTY
@@ -17,11 +17,11 @@ Comment:
 information or errors found in these notices.

 Files: *
-Copyright: 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright: 2011-2015 Dropbox, Inc.
 License: Apache-2

 Files: api/*
-Copyright: 2012-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright: 2012-2014 Dropbox, Inc
 License: Expat

 Files: api/integrations/perforce/git_p4.py
@@ -38,10 +38,6 @@ Files: confirmation/*
 Copyright: 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 License: BSD-3-Clause

-Files: docs/code-of-conduct.md
-Copyright: 2017, Kandra Labs Inc.
-License: CC-BY-SA-4.0
-
 Files: puppet/apt/*
 Copyright: 2011, Evolving Web Inc.
 License: Expat
@@ -83,14 +79,14 @@ Copyright: 2011, PagerDuty, Inc.
 License: BSD-3-Clause

 Files: puppet/zulip_internal/files/zulip-ec2-configure-interfaces
-Copyright: 2013-2017, Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright: 2013, Dropbox, Inc.
 License: Expat

 Files: static/audio/zulip.*
 Copyright: 2011 Vidsyn
 License: CC-0-1.0

-Files: static/third/thirdparty-fonts.css
+Files: static/styles/thirdparty-fonts.css
 Copyright: 2012-2013 Dave Gandy
 License: Expat

@@ -119,9 +115,10 @@ Copyright: 2013 Nijiko Yonskai
 License: Apache-2.0
 Comment: The software has been modified.

-Files: static/generated/emoji/images/emoji/unicode/* tools/setup/emoji/*.ttf
+Files: static/third/gemoji/images/emoji/unicode/* tools/setup/emoji_dump/*.ttf
 Copyright: Google, Inc.
 License: Apache-2.0
+Comment: These are actually Noto Emoji, not gemoji.

 Files: static/third/html5-formdata/formdata.js
 Copyright: 2010 François de Metz
@@ -166,6 +163,10 @@ Files: static/third/jquery-throttle-debounce/*
 Copyright: 2010 "Cowboy" Ben Alman
 License: Expat or GPL

+Files: static/third/jquery-validate/*
+Copyright: 2006 - 2011 Jörn Zaefferer
+License: Expat
+
 Files: src/zulip/static/third/lazyload/*
 Copyright: 2011 Ryan Grove
 License: Expat
@@ -174,6 +175,10 @@ Files: static/third/marked/*
 Copyright: 2011-2013, Christopher Jeffrey
 License: Expat

+Files: static/third/string-prototype-codepointat/*
+Copyright: 2014 Mathias Bynens
+License: Expat
+
 Files: static/third/sockjs/sockjs-0.3.4.js
 Copyright: 2011-2012 VMware, Inc.
 2012 Douglas Crockford
@@ -191,11 +196,41 @@ Files: static/third/spectrum/*
 Copyright: 2013 Brian Grinstead
 License: Expat

+Files: static/third/spin/spin.js
+Copyright: 2011-2013 Felix Gnass
+License: Expat
+
+Files: static/third/underscore/underscore.js
+Copyright: 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+License: Expat
+Comment: https://github.com/jashkenas/underscore/blob/master/LICENSE
+
+Files: static/third/winchan/*
+Copyright: 2012 Lloyd Hilaiel
+License: Expat
+Comment: https://github.com/mozilla/winchan
+
+Files: static/third/xdate/*
+Copyright: 2010 C. F., Wong
+License: Expat
+
 Files: static/third/zocial/*
 Copyright: Sam Collins
 License: Expat

-Files: zerver/lib/bugdown/fenced_code.py
+Files: tools/inject-messages/othello
+Copyright: Shakespeare
+License: public-domain
+
+Files: tools/jslint/jslint.js
+Copyright: 2002 Douglas Crockford
+License: XXX-good-not-evil
+
+Files: tools/review
+Copyright: 2010 Ksplice, Inc.
+License: Apache-2.0
+
+Files: zerver/lib/bugdown/codehilite.py zerver/lib/bugdown/fenced_code.py
 Copyright: 2006-2008 Waylan Limberg
 License: BSD-3-Clause
 Comment: https://pypi.python.org/pypi/Markdown
@@ -208,6 +243,16 @@ Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/
 Copyright: Django Software Foundation and individual contributors
 License: BSD-3-Clause

+Files: frontend_tests/casperjs/*
+Copyright: 2011-2012 Nicolas Perriault
+ Joyent, Inc. and other Node contributors
+License: Expat
+
+Files: frontend_tests/casperjs/modules/vendors/*
+Copyright: 2011, Jeremy Ashkenas
+License: Expat
+
+
 License: Apache-2.0
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
--- a/45
+++ b/45
@@ -7,18 +7,6 @@ def command?(name)
  $?.success?
 end

-if Vagrant::VERSION == "1.8.7" then
-    path = `which curl`
-    if path.include?('/opt/vagrant/embedded/bin/curl') then
-        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
-             "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
-             "issue before provisioning. See "\
-             "https://github.com/mitchellh/vagrant/issues/7997 "\
-             "for reference."
-        exit
-    end
-end
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

  # For LXC. VirtualBox hosts use a different box, described below.
@@ -27,7 +15,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  # The Zulip development environment runs on 9991 on the guest.
  host_port = 9991
  http_proxy = https_proxy = no_proxy = ""
-  host_ip_addr = "127.0.0.1"

  config.vm.synced_folder ".", "/vagrant", disabled: true
  config.vm.synced_folder ".", "/srv/zulip"
@@ -43,12 +30,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
      when "HTTPS_PROXY"; https_proxy = value
      when "NO_PROXY"; no_proxy = value
      when "HOST_PORT"; host_port = value.to_i
-      when "HOST_IP_ADDR"; host_ip_addr = value
      end
    end
  end

-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
+  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: "127.0.0.1"

  if Vagrant.has_plugin?("vagrant-proxyconf")
    if http_proxy != ""
@@ -81,43 +67,18 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
    override.vm.box = "ubuntu/trusty64"
    # It's possible we can get away with just 1.5GB; more testing needed
    vb.memory = 2048
-    vb.cpus = 2
-  end
-
-  config.vm.provider "vmware_fusion" do |vb, override|
-    override.vm.box = "puphpet/ubuntu1404-x64"
-    vb.vmx["memsize"] = "2048"
-    vb.vmx["numvcpus"] = "2"
  end

 $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
-# If the host is running SELinux remount the /sys/fs/selinux directory as read only,
-# needed for apt-get to work.
-if [ -d "/sys/fs/selinux" ]; then
-  sudo mount -o remount,ro /sys/fs/selinux
-fi
-
-# Set default locale, this prevents errors if the user has another locale set.
-if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
-    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
-fi
-
-# Provision the development environment
 ln -nsf /srv/zulip ~/zulip
-/srv/zulip/tools/provision
-
-# Run any custom provision hooks the user has configured
-if [ -f /srv/zulip/tools/custom_provision ]; then
-  chmod +x /srv/zulip/tools/custom_provision
-  /srv/zulip/tools/custom_provision
-fi
+/usr/bin/python /srv/zulip/tools/provision.py | sudo tee -a /var/log/zulip_provision.log
 SCRIPT

  config.vm.provision "shell",
-    # We want provision to be run with the permissions of the vagrant user.
+    # We want provision.py to be run with the permissions of the vagrant user.
    privileged: false,
    inline: $provision_script
 end
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -1,536 +0,0 @@
-from django.conf import settings
-from django.db import connection, models
-from django.db.models import F
-
-from analytics.models import InstallationCount, RealmCount, \
-    UserCount, StreamCount, BaseCount, FillState, Anomaly, installation_epoch, \
-    last_successful_fill
-from zerver.models import Realm, UserProfile, Message, Stream, \
-    UserActivityInterval, RealmAuditLog, models
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, ceiling_to_day, \
-    ceiling_to_hour
-
-from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, Union
-
-from collections import defaultdict, OrderedDict
-from datetime import timedelta, datetime
-import logging
-import time
-
-## Logging setup ##
-
-log_format = '%(asctime)s %(levelname)-8s %(message)s'
-logging.basicConfig(format=log_format)
-
-formatter = logging.Formatter(log_format)
-file_handler = logging.FileHandler(settings.ANALYTICS_LOG_PATH)
-file_handler.setFormatter(formatter)
-
-logger = logging.getLogger("zulip.management")
-logger.setLevel(logging.INFO)
-logger.addHandler(file_handler)
-
-# You can't subtract timedelta.max from a datetime, so use this instead
-TIMEDELTA_MAX = timedelta(days=365*1000)
-
-## Class definitions ##
-
-class CountStat(object):
-    HOUR = 'hour'
-    DAY = 'day'
-    FREQUENCIES = frozenset([HOUR, DAY])
-
-    def __init__(self, property, data_collector, frequency, interval=None):
-        # type: (str, DataCollector, str, Optional[timedelta]) -> None
-        self.property = property
-        self.data_collector = data_collector
-        # might have to do something different for bitfields
-        if frequency not in self.FREQUENCIES:
-            raise AssertionError("Unknown frequency: %s" % (frequency,))
-        self.frequency = frequency
-        if interval is not None:
-            self.interval = interval
-        elif frequency == CountStat.HOUR:
-            self.interval = timedelta(hours=1)
-        else:  # frequency == CountStat.DAY
-            self.interval = timedelta(days=1)
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<CountStat: %s>" % (self.property,)
-
-class LoggingCountStat(CountStat):
-    def __init__(self, property, output_table, frequency):
-        # type: (str, Type[BaseCount], str) -> None
-        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
-
-class DependentCountStat(CountStat):
-    def __init__(self, property, data_collector, frequency, interval=None, dependencies=[]):
-        # type: (str, DataCollector, str, Optional[timedelta], List[str]) -> None
-        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
-        self.dependencies = dependencies
-
-class DataCollector(object):
-    def __init__(self, output_table, pull_function):
-        # type: (Type[BaseCount], Optional[Callable[[str, datetime, datetime], int]]) -> None
-        self.output_table = output_table
-        self.pull_function = pull_function
-
-## CountStat-level operations ##
-
-def process_count_stat(stat, fill_to_time):
-    # type: (CountStat, datetime) -> None
-    if stat.frequency == CountStat.HOUR:
-        time_increment = timedelta(hours=1)
-    elif stat.frequency == CountStat.DAY:
-        time_increment = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
-
-    if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
-    if fill_to_time.tzinfo is None:
-        raise ValueError("fill_to_time must be timezone aware: %s" % (fill_to_time,))
-
-    fill_state = FillState.objects.filter(property=stat.property).first()
-    if fill_state is None:
-        currently_filled = installation_epoch()
-        fill_state = FillState.objects.create(property=stat.property,
-                                              end_time=currently_filled,
-                                              state=FillState.DONE)
-        logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
-    elif fill_state.state == FillState.STARTED:
-        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
-        do_delete_counts_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - time_increment
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        logger.info("UNDO DONE %s" % (stat.property,))
-    elif fill_state.state == FillState.DONE:
-        currently_filled = fill_state.end_time
-    else:
-        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
-
-    if isinstance(stat, DependentCountStat):
-        for dependency in stat.dependencies:
-            dependency_fill_time = last_successful_fill(dependency)
-            if dependency_fill_time is None:
-                logger.warning("DependentCountStat %s run before dependency %s." %
-                               (stat.property, dependency))
-                return
-            fill_to_time = min(fill_to_time, dependency_fill_time)
-
-    currently_filled = currently_filled + time_increment
-    while currently_filled <= fill_to_time:
-        logger.info("START %s %s" % (stat.property, currently_filled))
-        start = time.time()
-        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
-        do_fill_count_stat_at_hour(stat, currently_filled)
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        end = time.time()
-        currently_filled = currently_filled + time_increment
-        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
-
-def do_update_fill_state(fill_state, end_time, state):
-    # type: (FillState, datetime, int) -> None
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
-
-# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is timezone aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    start_time = end_time - stat.interval
-    if not isinstance(stat, LoggingCountStat):
-        timer = time.time()
-        assert(stat.data_collector.pull_function is not None)
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
-        logger.info("%s run pull_function (%dms/%sr)" %
-                    (stat.property, (time.time()-timer)*1000, rows_added))
-    do_aggregate_to_summary_table(stat, end_time)
-
-def do_delete_counts_at_hour(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    if isinstance(stat, LoggingCountStat):
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.output_table in [UserCount, StreamCount]:
-            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-    else:
-        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-
-def do_aggregate_to_summary_table(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    cursor = connection.cursor()
-
-    # Aggregate into RealmCount
-    output_table = stat.data_collector.output_table
-    if output_table in (UserCount, StreamCount):
-        realmcount_query = """
-            INSERT INTO analytics_realmcount
-                (realm_id, value, property, subgroup, end_time)
-            SELECT
-                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
-                %(output_table)s.subgroup, %%(end_time)s
-            FROM zerver_realm
-            JOIN %(output_table)s
-            ON
-                zerver_realm.id = %(output_table)s.realm_id
-            WHERE
-                %(output_table)s.property = '%(property)s' AND
-                %(output_table)s.end_time = %%(end_time)s
-            GROUP BY zerver_realm.id, %(output_table)s.subgroup
-        """ % {'output_table': output_table._meta.db_table,
-               'property': stat.property}
-        start = time.time()
-        cursor.execute(realmcount_query, {'end_time': end_time})
-        end = time.time()
-        logger.info("%s RealmCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
-
-    # Aggregate into InstallationCount
-    installationcount_query = """
-        INSERT INTO analytics_installationcount
-            (value, property, subgroup, end_time)
-        SELECT
-            sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
-        FROM analytics_realmcount
-        WHERE
-            property = '%(property)s' AND
-            end_time = %%(end_time)s
-        GROUP BY analytics_realmcount.subgroup
-    """ % {'property': stat.property}
-    start = time.time()
-    cursor.execute(installationcount_query, {'end_time': end_time})
-    end = time.time()
-    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
-    cursor.close()
-
-## Utility functions called from outside counts.py ##
-
-# called from zerver/lib/actions.py; should not throw any errors
-def do_increment_logging_stat(zerver_object, stat, subgroup, event_time, increment=1):
-    # type: (Union[Realm, UserProfile, Stream], CountStat, Optional[Union[str, int, bool]], datetime, int) -> None
-    table = stat.data_collector.output_table
-    if table == RealmCount:
-        id_args = {'realm': zerver_object}
-    elif table == UserCount:
-        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
-    else:  # StreamCount
-        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
-
-    if stat.frequency == CountStat.DAY:
-        end_time = ceiling_to_day(event_time)
-    else:  # CountStat.HOUR:
-        end_time = ceiling_to_hour(event_time)
-
-    row, created = table.objects.get_or_create(
-        property=stat.property, subgroup=subgroup, end_time=end_time,
-        defaults={'value': increment}, **id_args)
-    if not created:
-        row.value = F('value') + increment
-        row.save(update_fields=['value'])
-
-def do_drop_all_analytics_tables():
-    # type: () -> None
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-    Anomaly.objects.all().delete()
-
-## DataCollector-level operations ##
-
-def do_pull_by_sql_query(property, start_time, end_time, query, group_by):
-    # type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int
-    if group_by is None:
-        subgroup = 'NULL'
-        group_by_clause  = ''
-    else:
-        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
-        group_by_clause = ', ' + subgroup
-
-    # We do string replacement here because cursor.execute will reject a
-    # group_by_clause given as a param.
-    # We pass in the datetimes as params to cursor.execute so that we don't have to
-    # think about how to convert python datetimes to SQL datetimes.
-    query_ = query % {'property': property, 'subgroup': subgroup,
-                      'group_by_clause': group_by_clause}
-    cursor = connection.cursor()
-    cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
-    rowcount = cursor.rowcount
-    cursor.close()
-    return rowcount
-
-def sql_data_collector(output_table, query, group_by):
-    # type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector
-    def pull_function(property, start_time, end_time):
-        # type: (str, datetime, datetime) -> int
-        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
-    return DataCollector(output_table, pull_function)
-
-def do_pull_minutes_active(property, start_time, end_time):
-    # type: (str, datetime, datetime) -> int
-    user_activity_intervals = UserActivityInterval.objects.filter(
-        end__gt=start_time, start__lt=end_time
-    ).select_related(
-        'user_profile'
-    ).values_list(
-        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
-
-    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
-    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        start = max(start_time, interval_start)
-        end = min(end_time, interval_end)
-        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
-
-    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
-                      end_time=end_time, value=int(seconds // 60))
-            for ids, seconds in seconds_active.items() if seconds >= 60]
-    UserCount.objects.bulk_create(rows)
-    return len(rows)
-
-count_message_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_message
-    ON
-        zerver_userprofile.id = zerver_message.sender_id
-    WHERE
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-# Note: ignores the group_by / group_by_clause.
-count_message_type_by_user_query = """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type = 1 THEN 'private_message'
-             WHEN
-                  zerver_recipient.type = 3 THEN 'huddle_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
-
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.pub_date >= %%(time_start)s AND
-            zerver_message.pub_date < %%(time_end)s
-        JOIN zerver_recipient
-        ON
-            zerver_message.recipient_id = zerver_recipient.id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_recipient.type_id = zerver_stream.id
-        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-
-# This query joins to the UserProfile table since all current queries that
-# use this also subgroup on UserProfile.is_bot. If in the future there is a
-# stat that counts messages by stream and doesn't need the UserProfile
-# table, consider writing a new query for efficiency.
-count_message_by_stream_query = """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-        zerver_stream.id = zerver_recipient.type_id
-    JOIN zerver_message
-    ON
-        zerver_recipient.id = zerver_message.recipient_id
-    JOIN zerver_userprofile
-    ON
-        zerver_message.sender_id = zerver_userprofile.id
-    WHERE
-        zerver_stream.date_created < %%(time_end)s AND
-        zerver_recipient.type = 2 AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_stream.id %(group_by_clause)s
-"""
-
-# Hardcodes the query needed by active_users:is_bot:day, since that is
-# currently the only stat that uses this.
-count_user_by_realm_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realm
-    JOIN zerver_userprofile
-    ON
-        zerver_realm.id = zerver_userprofile.realm_id
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
-        zerver_userprofile.date_joined >= %%(time_start)s AND
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_userprofile.is_active = TRUE
-    GROUP BY zerver_realm.id %(group_by_clause)s
-"""
-
-# Currently hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in ['user_created', 'user_deactivated', etc].
-# In particular, it's important to ensure that migrations don't cause that to happen.
-check_realmauditlog_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realmauditlog ral1
-    JOIN (
-        SELECT modified_user_id, max(event_time) AS max_event_time
-        FROM zerver_realmauditlog
-        WHERE
-            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
-            event_time < %%(time_end)s
-        GROUP BY modified_user_id
-    ) ral2
-    ON
-        ral1.event_time = max_event_time AND
-        ral1.modified_user_id = ral2.modified_user_id
-    JOIN zerver_userprofile
-    ON
-        ral1.modified_user_id = zerver_userprofile.id
-    WHERE
-        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
-"""
-
-check_useractivityinterval_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_useractivityinterval
-    ON
-        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
-    WHERE
-        zerver_useractivityinterval.end >= %%(time_start)s AND
-        zerver_useractivityinterval.start < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-count_realm_active_humans_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
-    FROM (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = 'active_users_audit:is_bot:day' AND
-            subgroup = 'false' AND
-            end_time = %%(time_end)s
-    ) usercount1
-    JOIN (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = '15day_actives::day' AND
-            end_time = %%(time_end)s
-    ) usercount2
-    ON
-        usercount1.user_id = usercount2.user_id
-    GROUP BY usercount1.realm_id
-"""
-
-# Currently unused and untested
-count_stream_by_realm_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realm
-    JOIN zerver_stream
-    ON
-        zerver_realm.id = zerver_stream.realm_id AND
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
-        zerver_stream.date_created >= %%(time_start)s AND
-        zerver_stream.date_created < %%(time_end)s
-    GROUP BY zerver_realm.id %(group_by_clause)s
-"""
-
-## CountStat declarations ##
-
-count_stats_ = [
-    # Messages Sent stats
-    # Stats that count the number of messages sent in various ways.
-    # These are also the set of stats that read from the Message table.
-
-    CountStat('messages_sent:is_bot:hour',
-              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.HOUR),
-    CountStat('messages_sent:message_type:day',
-              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
-    CountStat('messages_sent:client:day',
-              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
-              CountStat.DAY),
-    CountStat('messages_in_stream:is_bot:day',
-              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-
-    # Number of Users stats
-    # Stats that count the number of active users in the UserProfile.is_active sense.
-
-    # 'active_users_audit:is_bot:day' is the canonical record of which users were
-    # active on which days (in the UserProfile.is_active sense).
-    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-    CountStat('active_users_audit:is_bot:day',
-              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-    # sum sequence of 'active_users_log:is_bot:day', for any realm that
-    # started after the latter stat was introduced.
-    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
-    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-    # approximation, e.g. if a user is deactivated between the end of the
-    # day and when this stat is run, they won't be counted. However, is the
-    # simplest of the three to inspect by hand.
-    CountStat('active_users:is_bot:day',
-              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
-              CountStat.DAY, interval=TIMEDELTA_MAX),
-
-    # User Activity stats
-    # Stats that measure user activity in the UserActivityInterval sense.
-
-    CountStat('15day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
-
-    # Dependent stats
-    # Must come after their dependencies.
-
-    # Canonical account of the number of active humans in a realm on each day.
-    DependentCountStat('realm_active_humans::day',
-                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
-                       CountStat.DAY,
-                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
-]
-
-COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
--- a/analytics/lib/fixtures.py
+++ b/analytics/lib/fixtures.py
@@ -1,70 +0,0 @@
-from __future__ import division, absolute_import
-
-from zerver.models import Realm, UserProfile, Stream, Message
-from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
-from analytics.lib.counts import CountStat
-from analytics.lib.time_utils import time_range
-
-from datetime import datetime
-from math import sqrt
-from random import gauss, random, seed
-from typing import List
-
-from six.moves import range, zip
-
-def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
-                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
-                              frequency=CountStat.DAY, partial_sum=False, random_seed=26):
-    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
-    """
-    Generate semi-realistic looking time series data for testing analytics graphs.
-
-    days -- Number of days of data. Is the number of data points generated if
-        frequency is CountStat.DAY.
-    business_hours_base -- Average value during a business hour (or day) at beginning of
-        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
-    non_business_hours_base -- The above, for non-business hours/days.
-    growth -- Ratio between average values at end of time series and beginning of time series.
-    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
-        point is unaffected by the previous point, and at 1 each point is a deterministic
-        function of the previous point.
-    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
-        the variance.
-    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
-    frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
-    random_seed -- Seed for random number generator.
-    """
-    if frequency == CountStat.HOUR:
-        length = days*24
-        seasonality = [non_business_hours_base] * 24 * 7
-        for day in range(5):
-            for hour in range(8):
-                seasonality[24*day + hour] = business_hours_base
-        holidays  = []
-        for i in range(days):
-            holidays.extend([random() < holiday_rate] * 24)
-    elif frequency == CountStat.DAY:
-        length = days
-        seasonality = [8*business_hours_base + 16*non_business_hours_base] * 5 + \
-                      [24*non_business_hours_base] * 2
-        holidays = [random() < holiday_rate for i in range(days)]
-    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
-    if length < 2:
-        raise AssertionError("Must be generating at least 2 data points. "
-                             "Currently generating %s" % (length,))
-    growth_base = growth ** (1. / (length-1))
-    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
-
-    seed(random_seed)
-    noise_scalars = [gauss(0, 1)]
-    for i in range(1, length):
-        noise_scalars.append(noise_scalars[-1]*autocorrelation + gauss(0, 1)*(1-autocorrelation))
-
-    values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
-              for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-    if partial_sum:
-        for i in range(1, length):
-            values[i] = values[i-1] + values[i]
-    return [max(v, 0) for v in values]
--- a/analytics/lib/time_utils.py
+++ b/analytics/lib/time_utils.py
@@ -1,29 +0,0 @@
-from zerver.lib.timestamp import floor_to_hour, floor_to_day, timestamp_to_datetime
-from analytics.lib.counts import CountStat
-
-from datetime import datetime, timedelta
-from typing import List, Optional
-
-# If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
-# If min_length is greater than 0, pads the list to the left.
-# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
-# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(start, end, frequency, min_length):
-    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
-    if frequency == CountStat.HOUR:
-        end = floor_to_hour(end)
-        step = timedelta(hours=1)
-    elif frequency == CountStat.DAY:
-        end = floor_to_day(end)
-        step = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
-
-    times = []
-    if min_length is not None:
-        start = min(start, end - (min_length-1)*step)
-    current = end
-    while current >= start:
-        times.append(current)
-        current -= step
-    return list(reversed(times))
--- a/analytics/management/commands/active_user_stats.py
+++ b/analytics/management/commands/active_user_stats.py
@@ -2,8 +2,7 @@ from __future__ import absolute_import
 from __future__ import print_function

 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-from typing import Any, Dict, List
+from typing import Any

 from zerver.models import UserPresence, UserActivity
 from zerver.lib.utils import statsd, statsd_key
@@ -19,13 +18,13 @@ class Command(BaseCommand):
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        # Get list of all active users in the last 1 week
-        cutoff = timezone_now() - timedelta(minutes=30, hours=168)
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)

        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)

        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
        hour_buckets = [0.16, 2, 12, 24, 48, 168]
-        user_info = defaultdict(dict)  # type: Dict[str, Dict[float, List[str]]]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]

        for last_presence in users:
            if last_presence.status == UserPresence.IDLE:
@@ -34,10 +33,10 @@ class Command(BaseCommand):
                known_active = last_presence.timestamp

            for bucket in hour_buckets:
-                if bucket not in user_info[last_presence.user_profile.realm.string_id]:
-                    user_info[last_presence.user_profile.realm.string_id][bucket] = []
-                if timezone_now() - known_active < timedelta(hours=bucket):
-                    user_info[last_presence.user_profile.realm.string_id][bucket].append(last_presence.user_profile.email)
+                if bucket not in user_info[last_presence.user_profile.realm.domain]:
+                    user_info[last_presence.user_profile.realm.domain][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)

        for realm, buckets in user_info.items():
            print("Realm %s" % (realm,))
@@ -50,10 +49,10 @@ class Command(BaseCommand):
        user_info = defaultdict(dict)
        for activity in users_reading:
            for bucket in hour_buckets:
-                if bucket not in user_info[activity.user_profile.realm.string_id]:
-                    user_info[activity.user_profile.realm.string_id][bucket] = []
-                if timezone_now() - activity.last_visit < timedelta(hours=bucket):
-                    user_info[activity.user_profile.realm.string_id][bucket].append(activity.user_profile.email)
+                if bucket not in user_info[activity.user_profile.realm.domain]:
+                    user_info[activity.user_profile.realm.domain][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
        for realm, buckets in user_info.items():
            print("Realm %s" % (realm,))
            for hr, users in sorted(buckets.items()):
--- a/analytics/management/commands/active_user_stats_by_day.py
+++ b/analytics/management/commands/active_user_stats_by_day.py
@@ -6,25 +6,22 @@ import pytz

 from optparse import make_option
 from typing import Any
-
-from django.core.management.base import BaseCommand, CommandParser
-from django.utils.timezone import now as timezone_now
+from django.core.management.base import BaseCommand
 from zerver.lib.statistics import activity_averages_during_day

 class Command(BaseCommand):
    help = "Generate statistics on user activity for a given day."

-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
-        parser.add_argument('--date', default=None, action='store',
-                            help="Day to query in format 2013-12-05.  Default is yesterday")
+    option_list = BaseCommand.option_list + \
+        (make_option('--date', default=None, action='store',
+                     help="Day to query in format 2013-12-05.  Default is yesterday"),)

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options["date"] is None:
-            date = timezone_now() - datetime.timedelta(days=1)
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
        else:
-            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=pytz.utc)
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
        print("Activity data for", date)
        print(activity_averages_during_day(date))
        print("Please note that the total registered user count is a total for today")
--- a/analytics/management/commands/analyze_mit.py
+++ b/analytics/management/commands/analyze_mit.py
@@ -1,10 +1,10 @@
 from __future__ import absolute_import
 from __future__ import print_function

-from typing import Any, Dict
+from typing import Any

 from optparse import make_option
-from django.core.management.base import BaseCommand, CommandParser
+from django.core.management.base import BaseCommand
 from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
 import datetime
@@ -17,7 +17,7 @@ def compute_stats(log_level):
    logger.setLevel(log_level)

    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
+    mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
                                       recipient__type=Recipient.STREAM,
                                       pub_date__gt=one_week_ago)
    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
@@ -30,15 +30,15 @@ def compute_stats(log_level):
                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                           "root@mit.edu", "nagios@mit.edu",
                           "www-data|local-realm@mit.edu"])
-    user_counts = {}  # type: Dict[str, Dict[str, int]]
+    user_counts = {} # type: Dict[str, Dict[str, int]]
    for m in mit_query.select_related("sending_client", "sender"):
        email = m.sender.email
        user_counts.setdefault(email, {})
        user_counts[email].setdefault(m.sending_client.name, 0)
        user_counts[email][m.sending_client.name] += 1

-    total_counts = {}  # type: Dict[str, int]
-    total_user_counts = {}  # type: Dict[str, int]
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
    for email, counts in user_counts.items():
        total_user_counts.setdefault(email, 0)
        for client_name, count in counts.items():
@@ -47,13 +47,13 @@ def compute_stats(log_level):
            total_user_counts[email] += count

    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}  # type: Dict[int, float]
+    top_percents = {} # type: Dict[int, float]
    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
        top_percents[size] = 0.0
    for i, email in enumerate(sorted(total_user_counts.keys(),
                                     key=lambda x: -total_user_counts[x])):
        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
-                              total_user_counts[email], 1)
+                               total_user_counts[email], 1)
        for size in top_percents.keys():
            top_percents.setdefault(size, 0)
            if i < size:
@@ -73,11 +73,10 @@ def compute_stats(log_level):
        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))

 class Command(BaseCommand):
-    help = "Compute statistics on MIT Zephyr usage."
+    option_list = BaseCommand.option_list + \
+        (make_option('--verbose', default=False, action='store_true'),)

-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
-        parser.add_argument('--verbose', default=False, action='store_true')
+    help = "Compute statistics on MIT Zephyr usage."

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
--- a/analytics/management/commands/analyze_user_activity.py
+++ b/analytics/management/commands/analyze_user_activity.py
@@ -7,7 +7,7 @@ from typing import Any, Dict
 from zerver.lib.statistics import seconds_usage_between

 from optparse import make_option
-from django.core.management.base import BaseCommand, CommandParser
+from django.core.management.base import BaseCommand
 from zerver.models import UserProfile
 import datetime
 from django.utils.timezone import utc
@@ -19,7 +19,7 @@ def analyze_activity(options):

    user_profile_query = UserProfile.objects.all()
    if options["realm"]:
-        user_profile_query = user_profile_query.filter(realm__string_id=options["realm"])
+        user_profile_query = user_profile_query.filter(realm__domain=options["realm"])

    print("Per-user online duration:\n")
    total_duration = datetime.timedelta(0)
@@ -47,17 +47,17 @@ It will correctly not count server-initiated reloads in the activity statistics.

 The duration flag can be used to control how many days to show usage duration for

-Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--duration=1]
+Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]

 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""

-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
-        parser.add_argument('--realm', action='store')
-        parser.add_argument('--date', action='store', default="2013-09-06")
-        parser.add_argument('--duration', action='store', default=1, type=int,
-                            help="How many days to show usage information for")
+    option_list = BaseCommand.option_list + (
+        make_option('--realm', action='store'),
+        make_option('--date', action='store', default="2013-09-06"),
+        make_option('--duration', action='store', default=1, type=int,
+                    help="How many days to show usage information for"),
+        )

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
--- a/analytics/management/commands/clear_analytics_tables.py
+++ b/analytics/management/commands/clear_analytics_tables.py
@@ -1,29 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-
-import sys
-
-from argparse import ArgumentParser
-from django.db import connection
-from django.core.management.base import BaseCommand
-
-from analytics.lib.counts import do_drop_all_analytics_tables
-
-from typing import Any
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser):
-        # type: (ArgumentParser) -> None
-        parser.add_argument('--force',
-                            action='store_true',
-                            help="Clear analytics tables.")
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        if options['force']:
-            do_drop_all_analytics_tables()
-        else:
-            print("Would delete all data from analytics tables (!); use --force to do so.")
-            sys.exit(1)
--- a/analytics/management/commands/client_activity.py
+++ b/analytics/management/commands/client_activity.py
@@ -6,7 +6,6 @@ from typing import Any
 from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
-from django.utils.timezone import now as timezone_now

 from zerver.models import UserActivity, UserProfile, Realm, \
    get_realm, get_user_profile_by_email
@@ -18,9 +17,9 @@ class Command(BaseCommand):

 Usage examples:

-./manage.py client_activity
-./manage.py client_activity zulip
-./manage.py client_activity hamlet@zulip.com"""
+python manage.py client_activity
+python manage.py client_activity zulip.com
+python manage.py client_activity jesstess@zulip.com"""

    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
@@ -39,7 +38,7 @@ Usage examples:
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
-        threshold = timezone_now() - datetime.timedelta(days=7)
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))
@@ -58,6 +57,7 @@ Usage examples:
            print("%25s %15d" % (count[1], count[0]))
        print("Total:", total)

+
    def handle(self, *args, **options):
        # type: (*Any, **str) -> None
        if options['arg'] is None:
@@ -69,13 +69,13 @@ Usage examples:
                # Report activity for a user.
                user_profile = get_user_profile_by_email(arg)
                self.compute_activity(UserActivity.objects.filter(
-                    user_profile=user_profile))
+                        user_profile=user_profile))
            except UserProfile.DoesNotExist:
                try:
                    # Report activity for a realm.
                    realm = get_realm(arg)
                    self.compute_activity(UserActivity.objects.filter(
-                        user_profile__realm=realm))
+                            user_profile__realm=realm))
                except Realm.DoesNotExist:
-                    print("Unknown user or realm %s" % (arg,))
+                    print("Unknown user or domain %s" % (arg,))
                    exit(1)
--- a/analytics/management/commands/populate_analytics_db.py
+++ b/analytics/management/commands/populate_analytics_db.py
@@ -1,138 +0,0 @@
-from __future__ import absolute_import, print_function
-
-from argparse import ArgumentParser
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
-from analytics.lib.fixtures import generate_time_series_data
-from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, FillState
-from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, UserProfile, Stream, Message, Client, \
-    RealmAuditLog
-
-from datetime import datetime, timedelta
-
-from six.moves import zip
-from typing import Any, Dict, List, Optional, Text, Type, Union, Mapping
-
-class Command(BaseCommand):
-    help = """Populates analytics tables with randomly generated data."""
-
-    DAYS_OF_DATA = 100
-    random_seed = 26
-
-    def create_user(self, email, full_name, is_staff, date_joined, realm):
-        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
-        user = UserProfile.objects.create(
-            email=email, full_name=full_name, is_staff=is_staff,
-            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
-            api_key='42', date_joined=date_joined)
-        RealmAuditLog.objects.create(
-            realm=realm, modified_user=user, event_type='user_created',
-            event_time=user.date_joined)
-        return user
-
-    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
-                              growth, autocorrelation, spikiness, holiday_rate=0,
-                              partial_sum=False):
-        # type: (CountStat, float, float, float, float, float, float, bool) -> List[int]
-        self.random_seed += 1
-        return generate_time_series_data(
-            days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
-            non_business_hours_base=non_business_hours_base, growth=growth,
-            autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        do_drop_all_analytics_tables()
-        # I believe this also deletes any objects with this realm as a foreign key
-        Realm.objects.filter(string_id='analytics').delete()
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
-        realm = Realm.objects.create(
-            string_id='analytics', name='Analytics', date_created=installation_time)
-        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
-
-        def insert_fixture_data(stat, fixture_data, table):
-            # type: (CountStat, Mapping[Optional[str], List[int]], Type[BaseCount]) -> None
-            end_times = time_range(last_end_time, last_end_time, stat.frequency,
-                                   len(list(fixture_data.values())[0]))
-            if table == RealmCount:
-                id_args = {'realm': realm}
-            if table == UserCount:
-                id_args = {'realm': realm, 'user': shylock}
-            for subgroup, values in fixture_data.items():
-                table.objects.bulk_create([
-                    table(property=stat.property, subgroup=subgroup, end_time=end_time,
-                          value=value, **id_args)
-                    for end_time, value in zip(end_times, values) if value != 0])
-
-        stat = COUNT_STATS['realm_active_humans::day']
-        realm_data = {
-            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data = {'false': self.generate_fixture_data(
-            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
-                      'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        user_data = {
-            'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
-            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
-            'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        website, created = Client.objects.get_or_create(name='website')
-        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
-        android, created = Client.objects.get_or_create(name='ZulipAndroid')
-        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
-        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
-        API, created = Client.objects.get_or_create(name='API: Python')
-        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
-        unused, created = Client.objects.get_or_create(name='unused')
-        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
-
-        stat = COUNT_STATS['messages_sent:client:day']
-        user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        # TODO: messages_sent_to_stream:is_bot
--- a/analytics/management/commands/realm_stats.py
+++ b/analytics/management/commands/realm_stats.py
@@ -2,7 +2,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

-from typing import Any, List
+from typing import Any

 from argparse import ArgumentParser
 import datetime
@@ -10,8 +10,6 @@ import pytz

 from django.core.management.base import BaseCommand
 from django.db.models import Count
-from django.utils.timezone import now as timezone_now
-
 from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
    Subscription, UserMessage, get_realm

@@ -31,27 +29,27 @@ class Command(BaseCommand):
    def active_users(self, realm):
        # type: (Realm) -> List[UserProfile]
        # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
-        return [activity.user_profile for activity in (
-            UserActivity.objects.filter(user_profile__realm=realm,
-                                        user_profile__is_active=True,
-                                        last_visit__gt=activity_cutoff,
-                                        query="/json/users/me/pointer",
-                                        client__name="website"))]
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
+        return [activity.user_profile for activity in \
+                    UserActivity.objects.filter(user_profile__realm=realm,
+                                                user_profile__is_active=True,
+                                                last_visit__gt=activity_cutoff,
+                                                query="/json/users/me/pointer",
+                                                client__name="website")]

    def messages_sent_by(self, user, days_ago):
        # type: (UserProfile, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()

    def total_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

    def human_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

    def api_messages(self, realm, days_ago):
@@ -60,19 +58,19 @@ class Command(BaseCommand):

    def stream_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                     recipient__type=Recipient.STREAM).count()

    def private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()

    def group_private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()

@@ -88,7 +86,7 @@ class Command(BaseCommand):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -96,7 +94,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)

            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            active_users = self.active_users(realm)
@@ -123,7 +121,7 @@ class Command(BaseCommand):
                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))

-            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
+            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications == True])
            self.report_percentage(num_notifications_enabled, num_active,
                                   "active users have desktop notifications enabled")

--- a/analytics/management/commands/stream_stats.py
+++ b/analytics/management/commands/stream_stats.py
@@ -20,7 +20,7 @@ class Command(BaseCommand):
        # type: (*Any, **str) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -28,7 +28,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)
            print("------------")
            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
--- a/analytics/management/commands/update_analytics_counts.py
+++ b/analytics/management/commands/update_analytics_counts.py
@@ -1,91 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-from scripts.lib.zulip_tools import ENDC, WARNING
-
-from argparse import ArgumentParser
-from datetime import timedelta
-import time
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-from django.utils.timezone import utc as timezone_utc
-from django.utils.dateparse import parse_datetime
-from django.conf import settings
-
-from analytics.models import RealmCount, UserCount
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import UserProfile, Message
-
-from typing import Any, Dict
-
-class Command(BaseCommand):
-    help = """Fills Analytics tables.
-
-    Run as a cron job that runs every hour."""
-
-    def add_arguments(self, parser):
-        # type: (ArgumentParser) -> None
-        parser.add_argument('--time', '-t',
-                            type=str,
-                            help='Update stat tables from current state to --time. Defaults to the current time.',
-                            default=timezone_now().isoformat())
-        parser.add_argument('--utc',
-                            action='store_true',
-                            help="Interpret --time in UTC.",
-                            default=False)
-        parser.add_argument('--stat', '-s',
-                            type=str,
-                            help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument('--verbose',
-                            action='store_true',
-                            help="Print timing information to stdout.",
-                            default=False)
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        try:
-            os.mkdir(settings.ANALYTICS_LOCK_DIR)
-        except OSError:
-            print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
-            return
-
-        try:
-            self.run_update_analytics_counts(options)
-        finally:
-            os.rmdir(settings.ANALYTICS_LOCK_DIR)
-
-    def run_update_analytics_counts(self, options):
-        # type: (Dict[str, Any]) -> None
-        fill_to_time = parse_datetime(options['time'])
-
-        if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
-        if fill_to_time.tzinfo is None:
-            raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
-
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
-
-        if options['stat'] is not None:
-            stats = [COUNT_STATS[options['stat']]]
-        else:
-            stats = list(COUNT_STATS.values())
-
-        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
-        if options['verbose']:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options['verbose']:
-                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
-                last = time.time()
-
-        if options['verbose']:
-            print("Finished updating analytics counts through %s in %.3fs" %
-                  (fill_to_time, time.time() - start))
-        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
--- a/analytics/management/commands/user_stats.py
+++ b/analytics/management/commands/user_stats.py
@@ -7,8 +7,6 @@ import pytz
 from typing import Any

 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
 from zerver.models import UserProfile, Realm, Stream, Message, get_realm
 from six.moves import range

@@ -22,15 +20,15 @@ class Command(BaseCommand):

    def messages_sent_by(self, user, week):
        # type: (UserProfile, int) -> int
-        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
-        end = timezone_now() - datetime.timedelta(days=week*7)
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -38,7 +36,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            print("%d users" % (len(user_profiles),))
            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
--- a/analytics/migrations/0001_initial.py
+++ b/analytics/migrations/0001_initial.py
@@ -1,113 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
-from django.conf import settings
-import zerver.lib.str_utils
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('zerver', '0030_realm_org_type'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Anomaly',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('info', models.CharField(max_length=1000)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='HuddleCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('huddle', models.ForeignKey(to='zerver.Recipient')),
-                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='InstallationCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='RealmCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(to='zerver.Realm')),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='StreamCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(to='zerver.Realm')),
-                ('stream', models.ForeignKey(to='zerver.Stream')),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='UserCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(to='zerver.Realm')),
-                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='huddlecount',
-            unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
-        ),
-    ]
--- a/analytics/migrations/0002_remove_huddlecount.py
+++ b/analytics/migrations/0002_remove_huddlecount.py
@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='huddlecount',
-            unique_together=set([]),
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='huddle',
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='user',
-        ),
-        migrations.DeleteModel(
-            name='HuddleCount',
-        ),
-    ]
--- a/analytics/migrations/0003_fillstate.py
+++ b/analytics/migrations/0003_fillstate.py
@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import zerver.lib.str_utils
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0002_remove_huddlecount'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='FillState',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('property', models.CharField(unique=True, max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('state', models.PositiveSmallIntegerField()),
-                ('last_modified', models.DateTimeField(auto_now=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-    ]
--- a/analytics/migrations/0004_add_subgroup.py
+++ b/analytics/migrations/0004_add_subgroup.py
@@ -1,34 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0003_fillstate'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='installationcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='realmcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='streamcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='usercount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-    ]
--- a/analytics/migrations/0005_alter_field_size.py
+++ b/analytics/migrations/0005_alter_field_size.py
@@ -1,54 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0004_add_subgroup'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='installationcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='installationcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-    ]
--- a/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
+++ b/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0005_alter_field_size'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-    ]
--- a/analytics/migrations/0007_remove_interval.py
+++ b/analytics/migrations/0007_remove_interval.py
@@ -1,48 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10.4 on 2017-01-16 20:50
-from __future__ import unicode_literals
-
-from django.conf import settings
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0006_add_subgroup_to_unique_constraints'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='installationcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='realmcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='streamcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='usercount',
-            name='interval',
-        ),
-    ]
--- a/analytics/migrations/0008_add_count_indexes.py
+++ b/analytics/migrations/0008_add_count_indexes.py
@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10.5 on 2017-02-01 22:28
-from __future__ import unicode_literals
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('zerver', '0050_userprofile_avatar_version'),
-        ('analytics', '0007_remove_interval'),
-    ]
-
-    operations = [
-        migrations.AlterIndexTogether(
-            name='realmcount',
-            index_together=set([('property', 'end_time')]),
-        ),
-        migrations.AlterIndexTogether(
-            name='streamcount',
-            index_together=set([('property', 'realm', 'end_time')]),
-        ),
-        migrations.AlterIndexTogether(
-            name='usercount',
-            index_together=set([('property', 'realm', 'end_time')]),
-        ),
-    ]
--- a/analytics/migrations/0009_remove_messages_to_stream_stat.py
+++ b/analytics/migrations/0009_remove_messages_to_stream_stat.py
@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-
-def delete_messages_sent_to_stream_stat(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent_to_stream:is_bot'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0008_add_count_indexes'),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]
--- a/analytics/migrations/0010_clear_messages_sent_values.py
+++ b/analytics/migrations/0010_clear_messages_sent_values.py
@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-def clear_message_sent_by_message_type_values(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent:message_type:day'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]
--- a/analytics/migrations/0011_clear_analytics_tables.py
+++ b/analytics/migrations/0011_clear_analytics_tables.py
@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-
-def clear_analytics_tables(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0010_clear_messages_sent_values'),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]
--- a/analytics/models.py
+++ b/analytics/models.py
@@ -1,109 +0,0 @@
-from django.db import models
-
-from zerver.models import Realm, UserProfile, Stream, Recipient
-from zerver.lib.str_utils import ModelReprMixin
-from zerver.lib.timestamp import floor_to_day
-
-import datetime
-
-from typing import Optional, Tuple, Union, Dict, Any, Text
-
-class FillState(ModelReprMixin, models.Model):
-    property = models.CharField(max_length=40, unique=True)  # type: Text
-    end_time = models.DateTimeField()  # type: datetime.datetime
-
-    # Valid states are {DONE, STARTED}
-    DONE = 1
-    STARTED = 2
-    state = models.PositiveSmallIntegerField()  # type: int
-
-    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
-
-# The earliest/starting end_time in FillState
-# We assume there is at least one realm
-def installation_epoch():
-    # type: () -> datetime.datetime
-    earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    return floor_to_day(earliest_realm_creation)
-
-def last_successful_fill(property):
-    # type: (str) -> Optional[datetime.datetime]
-    fillstate = FillState.objects.filter(property=property).first()
-    if fillstate is None:
-        return None
-    if fillstate.state == FillState.DONE:
-        return fillstate.end_time
-    return fillstate.end_time - datetime.timedelta(hours=1)
-
-# would only ever make entries here by hand
-class Anomaly(ModelReprMixin, models.Model):
-    info = models.CharField(max_length=1000)  # type: Text
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<Anomaly: %s... %s>" % (self.info, self.id)
-
-class BaseCount(ModelReprMixin, models.Model):
-    # Note: When inheriting from BaseCount, you may want to rearrange
-    # the order of the columns in the migration to make sure they
-    # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)  # type: Text
-    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[Text]
-    end_time = models.DateTimeField()  # type: datetime.datetime
-    value = models.BigIntegerField()  # type: int
-    anomaly = models.ForeignKey(Anomaly, null=True)  # type: Optional[Anomaly]
-
-    class Meta(object):
-        abstract = True
-
-class InstallationCount(BaseCount):
-
-    class Meta(object):
-        unique_together = ("property", "subgroup", "end_time")
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
-
-class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("realm", "property", "subgroup", "end_time")
-        index_together = ["property", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
-
-class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile)
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("user", "property", "subgroup", "end_time")
-        # This index dramatically improves the performance of
-        # aggregating from users to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
-
-class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream)
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("stream", "property", "subgroup", "end_time")
-        # This index dramatically improves the performance of
-        # aggregating from streams to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)
--- a/analytics/tests/test_counts.py
+++ b/analytics/tests/test_counts.py
--- a/analytics/tests/test_fixtures.py
+++ b/analytics/tests/test_fixtures.py
@@ -1,31 +0,0 @@
-from zerver.lib.test_classes import ZulipTestCase
-
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self):
-        # type: () -> None
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=1500,
-            growth=2, spikiness=0, frequency=CountStat.HOUR)
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assertEqual(len(data), 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500*2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=2000,
-            autocorrelation=1, frequency=CountStat.HOUR)
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])
--- a/analytics/tests/test_views.py
+++ b/analytics/tests/test_views.py
@@ -1,327 +0,0 @@
-from __future__ import absolute_import
-
-from django.utils.timezone import get_fixed_timezone, utc
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, \
-    datetime_to_timestamp
-from zerver.models import Realm, UserProfile, Client, get_realm, \
-    get_user_profile_by_email
-
-from analytics.lib.counts import CountStat, COUNT_STATS
-from analytics.lib.time_utils import time_range
-from analytics.models import RealmCount, UserCount, BaseCount, \
-    FillState, last_successful_fill
-from analytics.views import stats, get_chart_data, sort_by_totals, \
-    sort_client_labels, rewrite_client_arrays
-
-from datetime import datetime, timedelta
-import mock
-import ujson
-
-from six.moves import range
-from typing import List, Dict, Optional
-
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self):
-        # type: () -> None
-        self.user = get_user_profile_by_email('hamlet@zulip.com')
-        self.login(self.user.email)
-        result = self.client_get('/stats')
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip Analytics for", result)
-
-class TestGetChartData(ZulipTestCase):
-    def setUp(self):
-        # type: () -> None
-        self.realm = get_realm('zulip')
-        self.user = get_user_profile_by_email('hamlet@zulip.com')
-        self.login(self.user.email)
-        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
-                               for i in range(4)]
-        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
-                              for i in range(4)]
-
-    def data(self, i):
-        # type: (int) -> List[int]
-        return [0, 0, i, 0]
-
-    def insert_data(self, stat, realm_subgroups, user_subgroups):
-        # type: (CountStat, List[Optional[str]], List[str]) -> None
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create([
-            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                       value=100+i, realm=self.realm)
-            for i, subgroup in enumerate(realm_subgroups)])
-        UserCount.objects.bulk_create([
-            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                      value=200+i, realm=self.realm, user=self.user)
-            for i, subgroup in enumerate(user_subgroups)])
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'human': self.data(100)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_over_time(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        self.insert_data(stat, ['true', 'false'], ['false'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-            'frequency': CountStat.HOUR,
-            'realm': {'bot': self.data(100), 'human': self.data(101)},
-            'user': {'bot': self.data(0), 'human': self.data(200)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_message_type(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        self.insert_data(stat, ['public_stream', 'private_message'],
-                         ['public_stream', 'private_stream'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'Public streams': self.data(100), 'Private streams': self.data(0),
-                      'Private messages': self.data(101), 'Group private messages': self.data(0)},
-            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
-                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
-            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_client(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:client:day']
-        client1 = Client.objects.create(name='client 1')
-        client2 = Client.objects.create(name='client 2')
-        client3 = Client.objects.create(name='client 3')
-        client4 = Client.objects.create(name='client 4')
-        self.insert_data(stat, [client4.id, client3.id, client2.id],
-                         [client3.id, client1.id])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'client 4': self.data(100), 'client 3': self.data(101),
-                      'client 2': self.data(102)},
-            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
-            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
-            'result': 'success',
-        })
-
-    def test_include_empty_subgroups(self):
-        # type: () -> None
-        FillState.objects.create(
-            property='realm_active_humans::day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['realm'], {'human': [0]})
-        self.assertFalse('user' in data)
-
-        FillState.objects.create(
-            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['realm'], {'human': [0], 'bot': [0]})
-        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:message_type:day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['realm'], {
-            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
-        self.assertEqual(data['user'], {
-            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:client:day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['realm'], {})
-        self.assertEqual(data['user'], {})
-
-    def test_start_and_end(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[1],
-                                  'end': end_time_timestamps[2]})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
-        self.assertEqual(data['realm'], {'human': [0, 100]})
-
-        # start later then end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[2],
-                                  'end': end_time_timestamps[1]})
-        self.assert_json_error_contains(result, 'Start time is later than')
-
-    def test_min_length(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        # test min_length is too short to change anything
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 2})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
-        self.assertEqual(data['realm'], {'human': self.data(100)})
-        # test min_length larger than filled data
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 5})
-        self.assert_json_success(result)
-        data = ujson.loads(result.content)
-        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(data['realm'], {'human': [0]+self.data(100)})
-
-    def test_non_existent_chart(self):
-        # type: () -> None
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'does_not_exist'})
-        self.assert_json_error_contains(result, 'Unknown chart name')
-
-    def test_analytics_not_running(self):
-        # type: () -> None
-        # try to get data for a valid chart, but before we've put anything in the database
-        # (e.g. before update_analytics_counts has been run)
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'number_of_humans'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    # last_successful_fill is in analytics/models.py, but get_chart_data is
-    # the only function that uses it at the moment
-    def test_last_successful_fill(self):
-        # type: () -> None
-        self.assertIsNone(last_successful_fill('non-existant'))
-        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
-        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
-        fillstate = FillState.objects.create(property='property', end_time=a_time,
-                                             state=FillState.DONE)
-        self.assertEqual(last_successful_fill('property'), a_time)
-        fillstate.state = FillState.STARTED
-        fillstate.save()
-        self.assertEqual(last_successful_fill('property'), one_hour_before)
-
-    def test_sort_by_totals(self):
-        # type: () -> None
-        empty = []  # type: List[int]
-        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
-        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
-
-    def test_sort_client_labels(self):
-        # type: () -> None
-        data = {'realm': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
-                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
-        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
-
-class TestTimeRange(ZulipTestCase):
-    def test_time_range(self):
-        # type: () -> None
-        HOUR = timedelta(hours=1)
-        DAY = timedelta(days=1)
-        TZINFO = get_fixed_timezone(-100)  # 100 minutes west of UTC
-
-        # Using 22:59 so that converting to UTC and applying floor_to_{hour,day} do not commute
-        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)
-        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=TZINFO)
-        floor_day = datetime(2016, 3, 14).replace(tzinfo=TZINFO)
-
-        # test start == end
-        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
-        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
-        # test start == end == boundary, and min_length == 0
-        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
-        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
-        # test start and end on different boundaries
-        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, None),
-                         [floor_hour, floor_hour+HOUR])
-        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, None),
-                         [floor_day, floor_day+DAY])
-        # test min_length
-        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, 4),
-                         [floor_hour-2*HOUR, floor_hour-HOUR, floor_hour, floor_hour+HOUR])
-        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, 4),
-                         [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
-
-class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self):
-        # type: () -> None
-        a = {'desktop app 1.0': [1, 2, 3],
-             'desktop app 2.0': [10, 12, 13],
-             'desktop app 3.0': [21, 22, 23],
-             'website': [1, 2, 3],
-             'ZulipiOS': [1, 2, 3],
-             'ZulipMobile': [1, 5, 7],
-             'ZulipPython': [1, 2, 3],
-             'API: Python': [1, 2, 3],
-             'SomethingRandom': [4, 5, 6],
-             'ZulipGitHubWebhook': [7, 7, 9],
-             'ZulipAndroid': [64, 63, 65]}
-        result = rewrite_client_arrays(a)
-        self.assertEqual(result,
-                         {'Old desktop app': [32, 36, 39],
-                          'Old iOS app': [1, 2, 3],
-                          'New iOS app': [1, 5, 7],
-                          'Website': [1, 2, 3],
-                          'Python API': [2, 4, 6],
-                          'SomethingRandom': [4, 5, 6],
-                          'GitHub webhook': [7, 7, 9],
-                          'Android app': [64, 63, 65]})
--- a/analytics/urls.py
+++ b/analytics/urls.py
@@ -1,39 +1,9 @@
-from django.conf.urls import url, include
-from zerver.lib.rest import rest_dispatch
-
-import analytics.views
+from django.conf.urls import patterns, url

 i18n_urlpatterns = [
-    # Server admin (user_profile.is_staff) visible stats pages
-    url(r'^activity$', analytics.views.get_activity,
-        name='analytics.views.get_activity'),
-    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
-        name='analytics.views.get_realm_activity'),
-    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
-        name='analytics.views.get_user_activity'),
-
-    # User-visible stats page
-    url(r'^stats$', analytics.views.stats,
-        name='analytics.views.stats'),
+    url(r'^activity$', 'analytics.views.get_activity'),
+    url(r'^realm_activity/(?P<realm>[\S]+)/$', 'analytics.views.get_realm_activity'),
+    url(r'^user_activity/(?P<email>[\S]+)/$', 'analytics.views.get_user_activity'),
 ]

-# These endpoints are a part of the API (V1), which uses:
-# * REST verbs
-# * Basic auth (username:password is email:apiKey)
-# * Takes and returns json-formatted data
-#
-# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
-#
-# All of these paths are accessed by either a /json or /api prefix
-v1_api_and_json_patterns = [
-    # get data for the graphs at /stats
-    url(r'^analytics/chart_data$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data'}),
-]
-
-i18n_urlpatterns += [
-    url(r'^api/v1/', include(v1_api_and_json_patterns)),
-    url(r'^json/', include(v1_api_and_json_patterns)),
-]
-
-urlpatterns = i18n_urlpatterns
+urlpatterns = patterns('', *i18n_urlpatterns)
--- a/analytics/views.py
+++ b/analytics/views.py
@@ -1,209 +1,33 @@
-from __future__ import absolute_import, division
+from __future__ import absolute_import
+from __future__ import division
+from six import text_type
+from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union

-from django.conf import settings
-from django.core import urlresolvers
 from django.db import connection
-from django.db.models import Sum
 from django.db.models.query import QuerySet
-from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from django.template import RequestContext, loader
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import ugettext as _
-from django.shortcuts import render
+from django.core import urlresolvers
+from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from jinja2 import Markup as mark_safe

-from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS
-from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, last_successful_fill
-
-from zerver.decorator import has_request_variables, REQ, require_server_admin, \
-    zulip_login_required, to_non_negative_int, to_utc_datetime
-from zerver.lib.request import JsonableError
-from zerver.lib.response import json_success
-from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, timestamp_to_datetime
-from zerver.models import Realm, UserProfile, UserActivity, \
-    UserActivityInterval, Client
+from zerver.decorator import has_request_variables, REQ, zulip_internal
+from zerver.models import get_realm, UserActivity, UserActivityInterval, Realm
+from zerver.lib.timestamp import timestamp_to_datetime

 from collections import defaultdict
 from datetime import datetime, timedelta
 import itertools
-import json
-import logging
-import pytz
-import re
 import time
-
-from six.moves import filter, map, range, zip
-from typing import Any, Callable, Dict, List, Optional, Set, Text, \
-    Tuple, Type, Union
-
-@zulip_login_required
-def stats(request):
-    # type: (HttpRequest) -> HttpResponse
-    return render(request,
-                  'analytics/stats.html',
-                  context=dict(realm_name = request.user.realm.name))
-
-@has_request_variables
-def get_chart_data(request, user_profile, chart_name=REQ(),
-                   min_length=REQ(converter=to_non_negative_int, default=None),
-                   start=REQ(converter=to_utc_datetime, default=None),
-                   end=REQ(converter=to_utc_datetime, default=None)):
-    # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse
-    if chart_name == 'number_of_humans':
-        stat = COUNT_STATS['realm_active_humans::day']
-        tables = [RealmCount]
-        subgroup_to_label = {None: 'human'}  # type: Dict[Optional[str], str]
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_over_time':
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        tables = [RealmCount, UserCount]
-        subgroup_to_label = {'false': 'human', 'true': 'bot'}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_by_message_type':
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        tables = [RealmCount, UserCount]
-        subgroup_to_label = {'public_stream': 'Public streams',
-                             'private_stream': 'Private streams',
-                             'private_message': 'Private messages',
-                             'huddle_message': 'Group private messages'}
-        labels_sort_function = lambda data: sort_by_totals(data['realm'])
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_by_client':
-        stat = COUNT_STATS['messages_sent:client:day']
-        tables = [RealmCount, UserCount]
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {str(id): name for id, name in Client.objects.values_list('id', 'name')}
-        labels_sort_function = sort_client_labels
-        include_empty_subgroups = False
-    else:
-        raise JsonableError(_("Unknown chart name: %s") % (chart_name,))
-
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None and end is not None and start > end:
-        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
-                            {'start': start, 'end': end})
-
-    realm = user_profile.realm
-    if start is None:
-        start = realm.date_created
-    if end is None:
-        end = last_successful_fill(stat.property)
-    if end is None or start > end:
-        logging.warning("User from realm %s attempted to access /stats, but the computed "
-                        "start time: %s (creation time of realm) is later than the computed "
-                        "end time: %s (last successful analytics update). Is the "
-                        "analytics cron job running?" % (realm.string_id, start, end))
-        raise JsonableError(_("No analytics data available. Please contact your server administrator."))
-
-    end_times = time_range(start, end, stat.frequency, min_length)
-    data = {'end_times': end_times, 'frequency': stat.frequency}
-    for table in tables:
-        if table == RealmCount:
-            data['realm'] = get_time_series_by_subgroup(
-                stat, RealmCount, realm.id, end_times, subgroup_to_label, include_empty_subgroups)
-        if table == UserCount:
-            data['user'] = get_time_series_by_subgroup(
-                stat, UserCount, user_profile.id, end_times, subgroup_to_label, include_empty_subgroups)
-    if labels_sort_function is not None:
-        data['display_order'] = labels_sort_function(data)
-    else:
-        data['display_order'] = None
-    return json_success(data=data)
-
-def sort_by_totals(value_arrays):
-    # type: (Dict[str, List[int]]) -> List[str]
-    totals = [(sum(values), label) for label, values in value_arrays.items()]
-    totals.sort(reverse=True)
-    return [label for total, label in totals]
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data):
-    # type: (Dict[str, Dict[str, List[int]]]) -> List[str]
-    realm_order = sort_by_totals(data['realm'])
-    user_order = sort_by_totals(data['user'])
-    label_sort_values = {}  # type: Dict[str, float]
-    for i, label in enumerate(realm_order):
-        label_sort_values[label] = i
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i-.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(),
-                                                  key=lambda x: x[1])]
-
-def table_filtered_to_id(table, key_id):
-    # type: (Type[BaseCount], int) -> QuerySet
-    if table == RealmCount:
-        return RealmCount.objects.filter(realm_id=key_id)
-    elif table == UserCount:
-        return UserCount.objects.filter(user_id=key_id)
-    elif table == StreamCount:
-        return StreamCount.objects.filter(stream_id=key_id)
-    elif table == InstallationCount:
-        return InstallationCount.objects.all()
-    else:
-        raise AssertionError("Unknown table: %s" % (table,))
-
-def client_label_map(name):
-    # type: (str) -> str
-    if name == "website":
-        return "Website"
-    if name.startswith("desktop app"):
-        return "Old desktop app"
-    if name == "ZulipAndroid":
-        return "Android app"
-    if name == "ZulipiOS":
-        return "Old iOS app"
-    if name == "ZulipMobile":
-        return "New iOS app"
-    if name in ["ZulipPython", "API: Python"]:
-        return "Python API"
-    if name.startswith("Zulip") and name.endswith("Webhook"):
-        return name[len("Zulip"):-len("Webhook")] + " webhook"
-    return name
-
-def rewrite_client_arrays(value_arrays):
-    # type: (Dict[str, List[int]]) -> Dict[str, List[int]]
-    mapped_arrays = {}  # type: Dict[str, List[int]]
-    for label, array in value_arrays.items():
-        mapped_label = client_label_map(label)
-        if mapped_label in mapped_arrays:
-            for i in range(0, len(array)):
-                mapped_arrays[mapped_label][i] += value_arrays[label][i]
-        else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
-    return mapped_arrays
-
-def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroup_to_label, include_empty_subgroups):
-    # type: (CountStat, Type[BaseCount], int, List[datetime], Dict[Optional[str], str], bool) -> Dict[str, List[int]]
-    queryset = table_filtered_to_id(table, key_id).filter(property=stat.property) \
-                                                  .values_list('subgroup', 'end_time', 'value')
-    value_dicts = defaultdict(lambda: defaultdict(int))  # type: Dict[Optional[str], Dict[datetime, int]]
-    for subgroup, end_time, value in queryset:
-        value_dicts[subgroup][end_time] = value
-    value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
-        if (subgroup in value_dicts) or include_empty_subgroups:
-            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
-
-    if stat == COUNT_STATS['messages_sent:client:day']:
-        # HACK: We rewrite these arrays to collapse the Client objects
-        # with similar names into a single sum, and generally give
-        # them better names
-        return rewrite_client_arrays(value_arrays)
-    return value_arrays
-
-
+import re
+import pytz
+from six.moves import filter
+from six.moves import map
+from six.moves import range
+from six.moves import zip
 eastern_tz = pytz.timezone('US/Eastern')

+from zproject.jinja2 import render_to_response
+
 def make_table(title, cols, rows, has_row_class=False):
    # type: (str, List[str], List[Any], bool) -> str

@@ -236,7 +60,7 @@ def get_realm_day_counts():
    # type: () -> Dict[str, Dict[str, str]]
    query = '''
        select
-            r.string_id,
+            r.domain,
            (now()::date - pub_date::date) age,
            count(*) cnt
        from zerver_message m
@@ -250,10 +74,10 @@ def get_realm_day_counts():
        and
            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
        group by
-            r.string_id,
+            r.domain,
            age
        order by
-            r.string_id,
+            r.domain,
            age
    '''
    cursor = connection.cursor()
@@ -261,13 +85,14 @@ def get_realm_day_counts():
    rows = dictfetchall(cursor)
    cursor.close()

-    counts = defaultdict(dict)  # type: Dict[str, Dict[int, int]]
+    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
    for row in rows:
-        counts[row['string_id']][row['age']] = row['cnt']
+        counts[row['domain']][row['age']] = row['cnt']
+

    result = {}
-    for string_id in counts:
-        raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
+    for domain in counts:
+        raw_cnts = [counts[domain].get(age, 0) for age in range(8)]
        min_cnt = min(raw_cnts)
        max_cnt = max(raw_cnts)

@@ -283,7 +108,7 @@ def get_realm_day_counts():
            return '<td class="number %s">%s</td>' % (good_bad, cnt)

        cnts = ''.join(map(format_count, raw_cnts))
-        result[string_id] = dict(cnts=cnts)
+        result[domain] = dict(cnts=cnts)

    return result

@@ -291,7 +116,7 @@ def realm_summary_table(realm_minutes):
    # type: (Dict[str, float]) -> str
    query = '''
        SELECT
-            realm.string_id,
+            realm.domain,
            coalesce(user_counts.active_user_count, 0) active_user_count,
            coalesce(at_risk_counts.at_risk_count, 0) at_risk_count,
            (
@@ -384,7 +209,7 @@ def realm_summary_table(realm_minutes):
                AND
                    last_visit > now() - interval '2 week'
        )
-        ORDER BY active_user_count DESC, string_id ASC
+        ORDER BY active_user_count DESC, domain ASC
        '''

    cursor = connection.cursor()
@@ -396,26 +221,26 @@ def realm_summary_table(realm_minutes):
    counts = get_realm_day_counts()
    for row in rows:
        try:
-            row['history'] = counts[row['string_id']]['cnts']
-        except Exception:
+            row['history'] = counts[row['domain']]['cnts']
+        except:
            row['history'] = ''

    # augment data with realm_minutes
    total_hours = 0.0
    for row in rows:
-        string_id = row['string_id']
-        minutes = realm_minutes.get(string_id, 0.0)
+        domain = row['domain']
+        minutes = realm_minutes.get(domain, 0.0)
        hours = minutes / 60.0
        total_hours += hours
        row['hours'] = str(int(hours))
        try:
            row['hours_per_user'] = '%.1f' % (hours / row['active_user_count'],)
-        except Exception:
+        except:
            pass

    # formatting
    for row in rows:
-        row['string_id'] = realm_activity_link(row['string_id'])
+        row['domain'] = realm_activity_link(row['domain'])

    # Count active sites
    def meets_goal(row):
@@ -435,8 +260,9 @@ def realm_summary_table(realm_minutes):
        total_bot_count += int(row['bot_count'])
        total_at_risk_count += int(row['at_risk_count'])

+
    rows.append(dict(
-        string_id='Total',
+        domain='Total',
        active_user_count=total_active_user_count,
        user_profile_count=total_user_profile_count,
        bot_count=total_bot_count,
@@ -469,20 +295,20 @@ def user_activity_intervals():
        'start',
        'end',
        'user_profile__email',
-        'user_profile__realm__string_id'
+        'user_profile__realm__domain'
    ).order_by(
-        'user_profile__realm__string_id',
+        'user_profile__realm__domain',
        'user_profile__email'
    )

-    by_string_id = lambda row: row.user_profile.realm.string_id
+    by_domain = lambda row: row.user_profile.realm.domain
    by_email = lambda row: row.user_profile.email

    realm_minutes = {}

-    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
+    for domain, realm_intervals in itertools.groupby(all_intervals, by_domain):
        realm_duration = timedelta(0)
-        output += '<hr>%s\n' % (string_id,)
+        output += '<hr>%s\n' % (domain,)
        for email, intervals in itertools.groupby(realm_intervals, by_email):
            duration = timedelta(0)
            for interval in intervals:
@@ -494,7 +320,7 @@ def user_activity_intervals():
            realm_duration += duration
            output += "  %-*s%s\n" % (37, email, duration)

-        realm_minutes[string_id] = realm_duration.total_seconds() / 60
+        realm_minutes[domain] = realm_duration.total_seconds() / 60

    output += "\nTotal Duration:                      %s\n" % (total_duration,)
    output += "\nTotal Duration in minutes:           %s\n" % (total_duration.total_seconds() / 60.,)
@@ -532,7 +358,7 @@ def sent_messages_report(realm):
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
-                r.string_id = %s
+                r.domain = %s
            and
                (not up.is_bot)
            and
@@ -551,7 +377,7 @@ def sent_messages_report(realm):
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
-                r.string_id = %s
+                r.domain = %s
            and
                up.is_bot
            and
@@ -586,7 +412,7 @@ def ad_hoc_queries():
                row[i] = fixup_func(row[i])

        for i, col in enumerate(cols):
-            if col == 'Realm':
+            if col == 'Domain':
                fix_rows(i, realm_activity_link)
            elif col in ['Last time', 'Last visit']:
                fix_rows(i, format_date_for_activity_reports)
@@ -607,7 +433,7 @@ def ad_hoc_queries():

        query = '''
            select
-                realm.string_id,
+                realm.domain,
                up.id user_id,
                client.name,
                sum(count) as hits,
@@ -618,13 +444,13 @@ def ad_hoc_queries():
            join zerver_realm realm on realm.id = up.realm_id
            where
                client.name like '%s'
-            group by string_id, up.id, client.name
+            group by domain, up.id, client.name
            having max(last_visit) > now() - interval '2 week'
-            order by string_id, up.id, client.name
+            order by domain, up.id, client.name
        ''' % (mobile_type,)

        cols = [
-            'Realm',
+            'Domain',
            'User id',
            'Name',
            'Hits',
@@ -639,7 +465,7 @@ def ad_hoc_queries():

    query = '''
        select
-            realm.string_id,
+            realm.domain,
            client.name,
            sum(count) as hits,
            max(last_visit) as last_time
@@ -649,13 +475,13 @@ def ad_hoc_queries():
        join zerver_realm realm on realm.id = up.realm_id
        where
            client.name like 'desktop%%'
-        group by string_id, client.name
+        group by domain, client.name
        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client.name
+        order by domain, client.name
    '''

    cols = [
-        'Realm',
+        'Domain',
        'Client',
        'Hits',
        'Last time'
@@ -665,11 +491,11 @@ def ad_hoc_queries():

    ###

-    title = 'Integrations by realm'
+    title = 'Integrations by domain'

    query = '''
        select
-            realm.string_id,
+            realm.domain,
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
@@ -687,13 +513,13 @@ def ad_hoc_queries():
            )
        or
            query like '%%external%%'
-        group by string_id, client_name
+        group by domain, client_name
        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client_name
+        order by domain, client_name
    '''

    cols = [
-        'Realm',
+        'Domain',
        'Client',
        'Hits',
        'Last time'
@@ -711,7 +537,7 @@ def ad_hoc_queries():
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
-            realm.string_id,
+            realm.domain,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
@@ -725,14 +551,14 @@ def ad_hoc_queries():
            )
        or
            query like '%%external%%'
-        group by client_name, string_id
+        group by client_name, domain
        having max(last_visit) > now() - interval '2 week'
-        order by client_name, string_id
+        order by client_name, domain
    '''

    cols = [
        'Client',
-        'Realm',
+        'Domain',
        'Hits',
        'Last time'
    ]
@@ -741,12 +567,12 @@ def ad_hoc_queries():

    return pages

-@require_server_admin
+@zulip_internal
@has_request_variables
 def get_activity(request):
    # type: (HttpRequest) -> HttpResponse
-    duration_content, realm_minutes = user_activity_intervals()  # type: Tuple[mark_safe, Dict[str, float]]
-    counts_content = realm_summary_table(realm_minutes)  # type: str
+    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
+    counts_content = realm_summary_table(realm_minutes) # type: str
    data = [
        ('Counts', counts_content),
        ('Durations', duration_content),
@@ -756,10 +582,10 @@ def get_activity(request):

    title = 'Activity'

-    return render(
-        request,
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, title=title, is_home=True),
+        dict(data=data, title=title, is_home=True),
+        request=request
    )

 def get_user_activity_records_for_realm(realm, is_bot):
@@ -774,9 +600,9 @@ def get_user_activity_records_for_realm(realm, is_bot):
    ]

    records = UserActivity.objects.filter(
-        user_profile__realm__string_id=realm,
-        user_profile__is_active=True,
-        user_profile__is_bot=is_bot
+            user_profile__realm__domain=realm,
+            user_profile__is_active=True,
+            user_profile__is_bot=is_bot
    )
    records = records.order_by("user_profile__email", "-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
@@ -793,7 +619,7 @@ def get_user_activity_records_for_email(email):
    ]

    records = UserActivity.objects.filter(
-        user_profile__email=email
+            user_profile__email=email
    )
    records = records.order_by("-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
@@ -811,10 +637,10 @@ def raw_user_activity_table(records):
    def row(record):
        # type: (QuerySet) -> List[Any]
        return [
-            record.query,
-            record.client.name,
-            record.count,
-            format_date_for_activity_reports(record.last_visit)
+                record.query,
+                record.client.name,
+                record.count,
+                format_date_for_activity_reports(record.last_visit)
        ]

    rows = list(map(row, records))
@@ -828,20 +654,19 @@ def get_user_activity_summary(records):
    #: We could use something like:
    # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
    #: but that would require this long `Union` to carry on throughout inner functions.
-    summary = {}  # type: Dict[str, Dict[str, Any]]
-
+    summary = {} # type: Dict[str, Dict[str, Any]]
    def update(action, record):
        # type: (str, QuerySet) -> None
        if action not in summary:
            summary[action] = dict(
-                count=record.count,
-                last_visit=record.last_visit
+                    count=record.count,
+                    last_visit=record.last_visit
            )
        else:
            summary[action]['count'] += record.count
            summary[action]['last_visit'] = max(
-                summary[action]['last_visit'],
-                record.last_visit
+                    summary[action]['last_visit'],
+                    record.last_visit
            )

    if records:
@@ -869,6 +694,7 @@ def get_user_activity_summary(records):
            update('pointer', record)
        update(client, record)

+
    return summary

 def format_date_for_activity_reports(date):
@@ -885,23 +711,23 @@ def user_activity_link(email):
    email_link = '<a href="%s">%s</a>' % (url, email)
    return mark_safe(email_link)

-def realm_activity_link(realm_str):
+def realm_activity_link(realm):
    # type: (str) -> mark_safe
    url_name = 'analytics.views.get_realm_activity'
-    url = urlresolvers.reverse(url_name, kwargs=dict(realm_str=realm_str))
-    realm_link = '<a href="%s">%s</a>' % (url, realm_str)
+    url = urlresolvers.reverse(url_name, kwargs=dict(realm=realm))
+    realm_link = '<a href="%s">%s</a>' % (url, realm)
    return mark_safe(realm_link)

 def realm_client_table(user_summaries):
    # type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
    exclude_keys = [
-        'internal',
-        'name',
-        'use',
-        'send',
-        'pointer',
-        'website',
-        'desktop',
+            'internal',
+            'name',
+            'use',
+            'send',
+            'pointer',
+            'website',
+            'desktop',
    ]

    rows = []
@@ -915,22 +741,22 @@ def realm_client_table(user_summaries):
            count = v['count']
            last_visit = v['last_visit']
            row = [
-                format_date_for_activity_reports(last_visit),
-                client,
-                name,
-                email_link,
-                count,
+                    format_date_for_activity_reports(last_visit),
+                    client,
+                    name,
+                    email_link,
+                    count,
            ]
            rows.append(row)

    rows = sorted(rows, key=lambda r: r[0], reverse=True)

    cols = [
-        'Last visit',
-        'Client',
-        'Name',
-        'Email',
-        'Count',
+            'Last visit',
+            'Client',
+            'Name',
+            'Email',
+            'Count',
    ]

    title = 'Clients'
@@ -947,25 +773,25 @@ def user_activity_summary_table(user_summary):
        count = v['count']
        last_visit = v['last_visit']
        row = [
-            format_date_for_activity_reports(last_visit),
-            client,
-            count,
+                format_date_for_activity_reports(last_visit),
+                client,
+                count,
        ]
        rows.append(row)

    rows = sorted(rows, key=lambda r: r[0], reverse=True)

    cols = [
-        'last_visit',
-        'client',
-        'count',
+            'last_visit',
+            'client',
+            'count',
    ]

    title = 'User Activity'
    return make_table(title, cols, rows)

 def realm_user_summary_table(all_records, admin_emails):
-    # type: (List[QuerySet], Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]
+    # type: (List[QuerySet], Set[text_type]) -> Tuple[Dict[str, Dict[str, Any]], str]
    user_records = {}

    def by_email(record):
@@ -991,7 +817,7 @@ def realm_user_summary_table(all_records, admin_emails):

    def is_recent(val):
        # type: (Optional[datetime]) -> bool
-        age = timezone_now() - val
+        age = datetime.now(val.tzinfo) - val # type: ignore # datetie.now tzinfo bug.
        return age.total_seconds() < 5 * 60

    rows = []
@@ -1013,21 +839,21 @@ def realm_user_summary_table(all_records, admin_emails):
        rows.append(row)

    def by_used_time(row):
-        # type: (Dict[str, Any]) -> str
+        # type: (Dict[str, Sequence[str]]) -> str
        return row['cells'][3]

    rows = sorted(rows, key=by_used_time, reverse=True)

    cols = [
-        'Name',
-        'Email',
-        'Total sent',
-        'Heard from',
-        'Message sent',
-        'Pointer motion',
-        'Desktop',
-        'ZulipiOS',
-        'Android',
+            'Name',
+            'Email',
+            'Total sent',
+            'Heard from',
+            'Message sent',
+            'Pointer motion',
+            'Desktop',
+            'ZulipiOS',
+            'Android'
    ]

    title = 'Summary'
@@ -1035,21 +861,21 @@ def realm_user_summary_table(all_records, admin_emails):
    content = make_table(title, cols, rows, has_row_class=True)
    return user_records, content

-@require_server_admin
-def get_realm_activity(request, realm_str):
+@zulip_internal
+def get_realm_activity(request, realm):
    # type: (HttpRequest, str) -> HttpResponse
-    data = []  # type: List[Tuple[str, str]]
-    all_user_records = {}  # type: Dict[str, Any]
+    data = [] # type: List[Tuple[str, str]]
+    all_user_records = {} # type: Dict[str, Any]

    try:
-        admins = Realm.objects.get(string_id=realm_str).get_admin_users()
+        admins = get_realm(realm).get_admin_users()
    except Realm.DoesNotExist:
-        return HttpResponseNotFound("Realm %s does not exist" % (realm_str,))
+        return HttpResponseNotFound("Realm %s does not exist" % (realm,))

    admin_emails = {admin.email for admin in admins}

-    for is_bot, page_title in [(False, 'Humans'), (True, 'Bots')]:
-        all_records = list(get_user_activity_records_for_realm(realm_str, is_bot))
+    for is_bot, page_title in [(False,  'Humans'), (True, 'Bots')]:
+        all_records = list(get_user_activity_records_for_realm(realm, is_bot))

        user_records, content = realm_user_summary_table(all_records, admin_emails)
        all_user_records.update(user_records)
@@ -1060,26 +886,29 @@ def get_realm_activity(request, realm_str):
    content = realm_client_table(all_user_records)
    data += [(page_title, content)]

+
    page_title = 'History'
-    content = sent_messages_report(realm_str)
+    content = sent_messages_report(realm)
    data += [(page_title, content)]

-    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
-    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (realm_str,)
+    fix_name = lambda realm: realm.replace('.', '_')

-    title = realm_str
-    return render(
-        request,
+    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
+    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (fix_name(realm),)
+
+    title = realm
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, realm_link=realm_link, title=title),
+        dict(data=data, realm_link=realm_link, title=title),
+        request=request
    )

-@require_server_admin
+@zulip_internal
 def get_user_activity(request, email):
    # type: (HttpRequest, str) -> HttpResponse
    records = get_user_activity_records_for_email(email)

-    data = []  # type: List[Tuple[str, str]]
+    data = [] # type: List[Tuple[str, str]]
    user_summary = get_user_activity_summary(records)
    content = user_activity_summary_table(user_summary)

@@ -1089,8 +918,8 @@ def get_user_activity(request, email):
    data += [('Info', content)]

    title = email
-    return render(
-        request,
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, title=title),
+        dict(data=data, title=title),
+        request=request
    )
--- a/api/README.md
+++ b/api/README.md
@@ -1,12 +1,11 @@
 #### Dependencies

-The [Zulip API](https://zulipchat.com/api) Python bindings require the
+The [Zulip API](https://zulip.com/api) Python bindings require the
 following Python libraries:

-* requests (version >= 0.12.1)
 * simplejson
-* six
-* typing (version >= 3.5.2.2)
+* requests (version >= 0.12.1)
+

 #### Installing

@@ -37,28 +36,22 @@ file is as follows:

 If omitted, these settings have the following defaults:

+    site=https://api.zulip.com
    insecure=false
    cert_bundle=<the default CA bundle trusted by Python>

-Alternatively, you may explicitly use "--user", "--api-key", and
-`--site` in our examples, which is especially useful when testing.  If
-you are running several bots which share a home directory, we
-recommend using `--config` to specify the path to the `zuliprc` file
-for a specific bot.  Finally, you can control the defaults for all of
-these variables using the environment variables `ZULIP_CONFIG`,
-`ZULIP_API_KEY`, `ZULIP_EMAIL`, `ZULIP_SITE`, `ZULIP_CERT`,
-`ZULIP_CERT_KEY`, and `ZULIP_CERT_BUNDLE`.  Command-line options take
-precedence over environment variables take precedence over the config
-files.
+Alternatively, you may explicitly use "--user" and "--api-key" in our
+examples, which is especially useful if you are running several bots
+which share a home directory.

 The command line equivalents for other configuration options are:

+    --site=<your Zulip server's URI>
    --insecure
    --cert-bundle=<file>

 You can obtain your Zulip API key, create bots, and manage bots all
-from your Zulip settings page; with current Zulip there's also a
-button to download a `zuliprc` file for your account/server pair.
+from your Zulip [settings page](https://zulip.com/#settings).

 A typical simple bot sending API messages will look as follows:

@@ -78,9 +71,6 @@ When you want to send a message:
    }
    zulip_client.send_message(message)

-If you are parsing arguments, you may find it useful to use Zulip's
-option group; see any of our API examples for details on how to do this.
-
 Additional examples:

    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
@@ -93,14 +83,7 @@ keys: msg, result.  For successful calls, result will be "success" and
 msg will be the empty string.  On error, result will be "error" and
 msg will describe what went wrong.

-#### Examples
-
-The API bindings package comes with several nice example scripts that
-show how to use the APIs; they are installed as part of the API
-bindings bundle.
-
 #### Logging
-
 The Zulip API comes with a ZulipStream class which can be used with the
 logging module:

@@ -129,7 +112,6 @@ Alternatively, if you don't want to use your ~/.zuliprc file:

    zulip-send --user shakespeare-bot@example.com \
        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
-        --site https://zulip.example.com \
        hamlet@example.com cordelia@example.com -m \
        "Conscience doth make cowards of us all."

--- a/api/bin/zulip-send
+++ b/api/bin/zulip-send
@@ -27,23 +27,17 @@ import os
 import optparse
 import logging

-from typing import Any, Dict, List, Optional
-
-sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-
-import zulip

 logging.basicConfig()

 log = logging.getLogger('zulip-send')

-def do_send_message(client, message_data):
-    # type: (zulip.Client, Dict[str, Any]) -> bool
+def do_send_message(client, message_data ):
    '''Sends a message and optionally prints status about the same.'''

    if message_data['type'] == 'stream':
-        log.info('Sending message to stream "%s", subject "%s"... ' %
-                 (message_data['to'], message_data['subject']))
+        log.info('Sending message to stream "%s", subject "%s"... ' % \
+            (message_data['to'], message_data['subject']))
    else:
        log.info('Sending message to %s... ' % message_data['to'])
    response = client.send_message(message_data)
@@ -55,7 +49,6 @@ def do_send_message(client, message_data):
        return False

 def main(argv=None):
-    # type: (Optional[List[str]]) -> int
    if argv is None:
        argv = sys.argv

@@ -70,6 +63,10 @@ def main(argv=None):
    '--user' and '--api-key' arguments.
    """

+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+    import zulip
+
    parser = optparse.OptionParser(usage=usage)

    # Grab parser options from the API common set
@@ -78,17 +75,18 @@ def main(argv=None):
    parser.add_option('-m', '--message',
                      help='Specifies the message to send, prevents interactive prompting.')

-    group = optparse.OptionGroup(parser, 'Stream parameters')  # type: ignore # https://github.com/python/typeshed/pull/1248
+    group = optparse.OptionGroup(parser, 'Stream parameters')
    group.add_option('-s', '--stream',
-                     dest='stream',
-                     action='store',
-                     help='Allows the user to specify a stream for the message.')
+                      dest='stream',
+                      action='store',
+                      help='Allows the user to specify a stream for the message.')
    group.add_option('-S', '--subject',
-                     dest='subject',
-                     action='store',
-                     help='Allows the user to specify a subject for the message.')
+                      dest='subject',
+                      action='store',
+                      help='Allows the user to specify a subject for the message.')
    parser.add_option_group(group)

+
    (options, recipients) = parser.parse_args(argv[1:])

    if options.verbose:
@@ -122,7 +120,7 @@ def main(argv=None):

    if not do_send_message(client, message_data):
        return 1
-    return 0
+

 if __name__ == '__main__':
    sys.exit(main())
--- a/api/bots/commute/commute.config
+++ b/api/bots/commute/commute.config
@@ -1,2 +0,0 @@
-[Google.com]
-api_key = abcdefghijklmnopqrstuvwxyz
--- a/api/bots/commute/commute.py
+++ b/api/bots/commute/commute.py
@@ -1,242 +0,0 @@
-from __future__ import print_function
-
-import datetime as dt
-import requests
-from os.path import expanduser
-from six.moves import configparser as cp
-
-home = expanduser('~')
-CONFIG_PATH = home + '/commute.config'
-
-class CommuteHandler(object):
-    '''
-    This plugin provides information regarding commuting
-    from an origin to a destination, providing a multitude of information.
-    It looks for messages starting with @mention of the bot.
-    '''
-
-    def __init__(self):
-        self.api_key = self.get_api_key()
-
-    def usage(self):
-        return '''
-            This plugin will allow briefings of estimated travel times,
-            distances and fare information for transit travel.
-            It can vary outputs depending on traffic conditions, departure and
-            arrival times as well as user preferences
-            (toll avoidance, preference for bus travel, etc.).
-            It looks for messages starting with @mention of the bot.
-
-            Users should input an origin and a destination
-            in any stream or through private messages to the bot to receive a
-            response in the same stream or through private messages if the
-            input was originally private.
-
-            Sample input:
-            @mention-botname origins=Chicago,IL,USA destinations=New+York,NY,USA
-            @mention-botname help
-            '''
-
-    help_info = '''
-    Obligatory Inputs:
-        Origin e.g. origins=New+York,NY,USA
-        Destination e.g. destinations=Chicago,IL,USA
-    Optional Inputs:
-        Mode Of Transport (inputs: driving, walking, bicycling, transit)
-        Default mode (no mode input) is driving
-        e.g. mode=driving or mode=transit
-        Units (metric or imperial)
-        e.g. units=metric
-        Restrictions (inputs: tolls, highways, ferries, indoor)
-        e.g. avoid=tolls
-        Departure Time (inputs: now or (YYYY, MM, DD, HH, MM, SS) departing)
-        e.g. departure_time=now or departure_time=2016,12,17,23,40,40
-        Arrival Time (inputs: (YYYY, MM, DD, HH, MM, SS) arriving)
-        e.g. arrival_time=2016,12,17,23,40,40
-        Languages:
-        Languages list: https://developers.google.com/maps/faq#languagesupport)
-        e.g. language=fr
-
-    Sample request:
-        @mention-botname origins=Chicago,IL,USA destinations=New+York,NY,USA language=fr
-
-    Please note:
-        Fare information can be derived, though is solely dependent on the
-        availability of the information
-        python run.py bots/followup/followup.py --config-file ~/.zuliprc-local
-        released by public transport operators.
-        Duration in traffic can only be derived if a departure time is set.
-        If a location has spaces in its name, please use a + symbol in the
-        place of the space/s.
-        A departure time and a arrival time can not be inputted at the same time
-        To add more than 1 input for a category,
-        e.g. more than 1 destinations,
-        use (|), e.g. destinations=Empire+State+Building|Statue+Of+Liberty
-        No spaces within addresses.
-        Departure times and arrival times must be in the UTC timezone,
-        you can use the timezone bot.
-                '''
-
-    # adds API Authentication Key to url request
-    def get_api_key(self):
-        # commute.config must be moved from
-        # ~/zulip/api/bots/commute/commute.config into
-        # ~/commute.config for program to work
-        # see readme.md for more information
-        with open(CONFIG_PATH) as settings:
-            config = cp.ConfigParser()
-            config.readfp(settings)
-            return config.get('Google.com', 'api_key')
-
-    # determines if bot will respond as a private message/ stream message
-    def send_info(self, message, letter, client):
-        client.send_reply(message, letter)
-
-    def calculate_seconds(self, time_str):
-        times = time_str.split(',')
-        times = [int(x) for x in times]
-        # UNIX time from January 1, 1970 00:00:00
-        unix_start_date = dt.datetime(1970, 1, 1, 0, 0, 0)
-        requested_time = dt.datetime(*times)
-        total_seconds = str(int((requested_time-unix_start_date)
-                                .total_seconds()))
-        return total_seconds
-
-    # adds departure time and arrival time paramaters into HTTP request
-    def add_time_to_params(self, params):
-        # limited to UTC timezone because of difficulty with user inputting
-        # correct string for input
-        if 'departure_time' in params:
-            if 'departure_time' != 'now':
-                params['departure_time'] = self.calculate_seconds(params['departure_time'])
-        elif 'arrival_time' in params:
-            params['arrival_time'] = self.calculate_seconds(params['arrival_time'])
-        return
-
-    # gets content for output and sends it to user
-    def get_send_content(self, rjson, params, message, client):
-        try:
-            # JSON list of output variables
-            variable_list = rjson["rows"][0]["elements"][0]
-            # determines if user has valid inputs
-            not_found = (variable_list["status"] == "NOT_FOUND")
-            invalid_request = (rjson["status"] == "INVALID_REQUEST")
-            no_result = (variable_list["status"] == "ZERO_RESULTS")
-
-            if no_result:
-                self.send_info(message,
-                               "Zero results\nIf stuck, try '@commute help'.",
-                               client)
-                return
-            elif not_found or invalid_request:
-                raise IndexError
-        except IndexError:
-            self.send_info(message,
-                           "Invalid input, please see instructions."
-                           "\nIf stuck, try '@commute help'.", client)
-            return
-
-        # origin and destination strings
-        begin = 'From: ' + rjson["origin_addresses"][0]
-        end = 'To: ' + rjson["destination_addresses"][0]
-        distance = 'Distance: ' + variable_list["distance"]["text"]
-        duration = 'Duration: ' + variable_list["duration"]["text"]
-        output = begin + '\n' + end + '\n' + distance
-        # if user doesn't know that default mode is driving
-        if 'mode' not in params:
-            mode = 'Mode of Transport: Driving'
-            output += '\n' + mode
-
-        # determines if fare information is available
-        try:
-            fare = ('Fare: ' + variable_list["fare"]["currency"] +
-                    variable_list["fare"]["text"])
-            output += '\n' + fare
-        except (KeyError, IndexError):
-            pass
-
-        # determines if traffic duration information is available
-        try:
-            traffic_duration = ('Duration in traffic: ' +
-                                variable_list["duration_in_traffic"]
-                                ["text"])
-            output += '\n' + traffic_duration
-        except (KeyError, IndexError):
-            output += '\n' + duration
-
-        # bot sends commute information to user
-        self.send_info(message, output, client)
-
-    # creates parameters for HTTP request
-    def parse_pair(self, content_list):
-        result = {}
-        for item in content_list:
-            # enables key value pair
-            org = item.split('=')
-            # ensures that invalid inputs are not entered into url request
-            if len(org) != 2:
-                continue
-            key, value = org
-            result[key] = value
-        return result
-
-    def receive_response(self, params, message, client):
-        def validate_requests(request):
-            if request.status_code == 200:
-                return request.json()
-            else:
-                self.send_info(message,
-                               "Something went wrong. Please try again." +
-                               " Error: {error_num}.\n{error_text}"
-                               .format(error_num=request.status_code,
-                                       error_text=request.text), client)
-                return
-        r = requests.get('https://maps.googleapis.com/maps/api/' +
-                         'distancematrix/json', params=params)
-        result = validate_requests(r)
-        return result
-
-    def handle_message(self, message, client, state_handler):
-        original_content = message['content']
-        query = original_content.split()
-
-        if "help" in query:
-            self.send_info(message, self.help_info, client)
-            return
-
-        params = self.parse_pair(query)
-        params['key'] = self.api_key
-        self.add_time_to_params(params)
-
-        rjson = self.receive_response(params, message, client)
-        if not rjson:
-            return
-
-        self.get_send_content(rjson, params, message, client)
-
-handler_class = CommuteHandler
-handler = CommuteHandler()
-
-def test_parse_pair():
-    result = handler.parse_pair(['departure_time=2016,12,20,23,59,00',
-                                'dog_foo=cat-foo'])
-    assert result == dict(departure_time='2016,12,20,23,59,00',
-                          dog_foo='cat-foo')
-
-def test_calculate_seconds():
-    result = handler.calculate_seconds('2016,12,20,23,59,00')
-    assert result == str(1482278340)
-
-def test_get_api_key():
-    # must change to your own api key for test to work
-    result = handler.get_api_key()
-    assert result == 'abcdefghijksm'
-
-def test_helper_functions():
-    test_parse_pair()
-    test_calculate_seconds()
-    test_get_api_key()
-
-if __name__ == '__main__':
-    test_helper_functions()
-    print('Success')
--- a/api/bots/commute/readme.md
+++ b/api/bots/commute/readme.md
@@ -1,72 +0,0 @@
-This bot will allow briefings of estimated travel times, distances and
-fare information for transit travel.
-
-It can respond to: departure times, arrival times, user preferences
-(toll avoidance, highway avoidance) and a mode of transport
-
-It can output: fare information, more detailed addresses on origin and
-destination, duration in traffic information, metric and imperical
-units and information in various languages.
-
-The bot will respond to the same stream input was in. And if called as
-private message, the bot will reply with a private message.
-
-To setup the bot, you will first need to move commute.config into
-the user home directory and add an API key.
-
-Move
-
-```
-~/zulip/api/bots/commute/commute.config
-```
-
-into
-
-```
-~/commute.config
-```
-
-To add an API key, please visit:
-https://developers.google.com/maps/documentation/distance-matrix/start
-to retrieve a key and copy your api key into commute.config
-
-Sample input and output:
-
-<pre><code>@commute help</code></pre>
-
-<pre><code>Obligatory Inputs:
-    Origin e.g. origins=New+York,NY,USA
-    Destination e.g. destinations=Chicago,IL,USA
-
-Optional Inputs:
-    Mode Of Transport (inputs: driving, walking, bicycling, transit)
-    Default mode (no mode input) is driving
-    e.g. mode=driving or mode=transit
-    Units (metric or imperial)
-    e.g. units=metric
-    Restrictions (inputs: tolls, highways, ferries, indoor)
-    e.g. avoid=tolls
-    Departure Time (inputs: now or (YYYY, MM, DD, HH, MM, SS) departing)
-    e.g. departure_time=now or departure_time=2016,12,17,23,40,40
-    Arrival Time (inputs: (YYYY, MM, DD, HH, MM, SS) arriving)
-    e.g. arrival_time=2016,12,17,23,40,40
-    Languages:
-    Languages list: https://developers.google.com/maps/faq#languagesupport)
-    e.g. language=fr
-</code></pre>
-
-Sample request:
-    <pre><code>
-    @commute origins=Chicago,IL,USA destinations=New+York,NY,USA language=fr
-    </code></pre>
-
-Please note:
-    Fare information can be derived, though is solely dependent on the
-    availability of the information released by public transport operators.
-    Duration in traffic can only be derived if a departure time is set.
-    If a location has spaces in its name, please use a + symbol in the
-    place of the space/s.
-    A departure time and a arrival time can not be inputted at the same time
-    No spaces within addresses.
-    Departure times and arrival times must be in the UTC timezone,
-    you can use the timezone bot.
--- a/api/bots/converter/assets/multiple-converts.png
+++ b/api/bots/converter/assets/multiple-converts.png
--- a/api/bots/converter/converter.py
+++ b/api/bots/converter/converter.py
@@ -1,130 +0,0 @@
-# See readme.md for instructions on running this code.
-
-from __future__ import absolute_import
-from __future__ import division
-
-import copy
-import importlib
-import sys
-from math import log10, floor
-
-import utils
-import re
-
-def is_float(value):
-    try:
-        float(value)
-        return True
-    except ValueError:
-        return False
-
-# Rounds the number 'x' to 'digits' significant digits.
-# A normal 'round()' would round the number to an absolute amount of
-# fractional decimals, e.g. 0.00045 would become 0.0.
-# 'round_to()' rounds only the digits that are not 0.
-# 0.00045 would then become 0.0005.
-def round_to(x, digits):
-    return round(x, digits-int(floor(log10(abs(x)))))
-
-class ConverterHandler(object):
-    '''
-    This plugin allows users to make conversions between various units,
-    e.g. Celsius to Fahrenheit, or kilobytes to gigabytes.
-    It looks for messages of the format
-    '@mention-bot <number> <unit_from> <unit_to>'
-    The message '@mention-bot help' posts a short description of how to use
-    the plugin, along with a list of all supported units.
-    '''
-
-    def usage(self):
-        return '''
-               This plugin allows users to make conversions between
-               various units, e.g. Celsius to Fahrenheit,
-               or kilobytes to gigabytes. It looks for messages of
-               the format '@mention-bot <number> <unit_from> <unit_to>'
-               The message '@mention-bot help' posts a short description of
-               how to use the plugin, along with a list of
-               all supported units.
-               '''
-
-    def handle_message(self, message, client, state_handler):
-        bot_response = get_bot_converter_response(message, client)
-        client.send_reply(message, bot_response)
-
-def get_bot_converter_response(message, client):
-    content = message['content']
-
-    words = content.lower().split()
-    convert_indexes = [i for i, word in enumerate(words) if word == "@convert"]
-    convert_indexes = [-1] + convert_indexes
-    results = []
-
-    for convert_index in convert_indexes:
-        if (convert_index + 1) < len(words) and words[convert_index + 1] == 'help':
-            results.append(utils.HELP_MESSAGE)
-            continue
-        if (convert_index + 3) < len(words):
-            number = words[convert_index + 1]
-            unit_from = utils.ALIASES.get(words[convert_index + 2], words[convert_index + 2])
-            unit_to = utils.ALIASES.get(words[convert_index + 3], words[convert_index + 3])
-            exponent = 0
-
-            if not is_float(number):
-                results.append(number + ' is not a valid number. ' + utils.QUICK_HELP)
-                continue
-
-            number = float(number)
-            number_res = copy.copy(number)
-
-            for key, exp in utils.PREFIXES.items():
-                if unit_from.startswith(key):
-                    exponent += exp
-                    unit_from = unit_from[len(key):]
-                if unit_to.startswith(key):
-                    exponent -= exp
-                    unit_to = unit_to[len(key):]
-
-            uf_to_std = utils.UNITS.get(unit_from, False)
-            ut_to_std = utils.UNITS.get(unit_to, False)
-
-            if uf_to_std is False:
-                results.append(unit_from + ' is not a valid unit. ' + utils.QUICK_HELP)
-            if ut_to_std is False:
-                results.append(unit_to + ' is not a valid unit.' + utils.QUICK_HELP)
-            if uf_to_std is False or ut_to_std is False:
-                continue
-
-            base_unit = uf_to_std[2]
-            if uf_to_std[2] != ut_to_std[2]:
-                unit_from = unit_from.capitalize() if uf_to_std[2] == 'kelvin' else unit_from
-                results.append(unit_to.capitalize() + ' and ' + unit_from +
-                               ' are not from the same category. ' + utils.QUICK_HELP)
-                continue
-
-            # perform the conversion between the units
-            number_res *= uf_to_std[1]
-            number_res += uf_to_std[0]
-            number_res -= ut_to_std[0]
-            number_res /= ut_to_std[1]
-
-            if base_unit == 'bit':
-                number_res *= 1024 ** (exponent // 3)
-            else:
-                number_res *= 10 ** exponent
-            number_res = round_to(number_res, 7)
-
-            results.append('{} {} = {} {}'.format(number,
-                                                  words[convert_index + 2],
-                                                  number_res,
-                                                  words[convert_index + 3]))
-
-        else:
-            results.append('Too few arguments given. ' + utils.QUICK_HELP)
-
-    new_content = ''
-    for idx, result in enumerate(results, 1):
-        new_content += ((str(idx) + '. conversion: ') if len(results) > 1 else '') + result + '\n'
-
-    return new_content
-
-handler_class = ConverterHandler
--- a/api/bots/converter/readme.md
+++ b/api/bots/converter/readme.md
@@ -1,70 +0,0 @@
-# Converter bot
-
-This bot allows users to perform conversions for various measurement units.
-
-## Usage
-
-Run this bot as described in [here](http://zulip.readthedocs.io/en/latest/bots-guide.html#how-to-deploy-a-bot).
-
-Use this bot with the following command
-
-`@convert <number> <unit_from> <unit_to>`
-
-This will convert `number`, given in the unit `unit_from`, to the unit `unit_to`
-and print the result.
-
-* `number` can be any floating-point number, e.g. 12, 13.05, 0.002.
-* `unit_from` and `unit_to` are two units from [the following](#supported-units) table in the same category.
-* `unit_from` and `unit_to` can be preceded by [these](#supported-prefixes) prefixes.
-
-### Supported units
-
-| Category | Units |
-| ----------------- | ----- |
-| Area | square-centimeter (cm^2, cm2), square-decimeter (dm^2, dm2), square-meter (m^2, m2), square-kilometer (km^2, km2), square-inch (in^2, in2), square-foot (ft^2, ft2), square-yard (y^2, y2), square-mile (mi^2, mi2), are (a), hectare (ha), acre (ac) |
-| Information | bit, byte |
-| Length | centimeter (cm), decimeter (dm), meter (m), kilometer (km), inch (in), foot (ft), yard (y), mile (mi), nautical-mile (nmi) |
-| Temperature | Kelvin (K), Celsius (C), Fahrenheit (F) |
-| Volume | cubic-centimeter (cm^3, cm3), cubic-decimeter (dm^3, dm3), liter (l), cubic-meter (m^3, m3), cubic-inch (in^3, in3), fluid-ounce (fl-oz), cubic-foot (ft^3, ft3), cubic-yard (y^3, y3) |
-| Weight | gram (g), kilogram (kg), ton (t), ounce (oz), pound (lb) |
-| Cooking (metric only, U.S. and imperial units differ slightly) | teaspoon (tsp), tablespoon (tbsp), cup |
-
-### Supported prefixes
-
-| Prefix | Power of 10 |
-| ------ | ----------- |
-| atto | 10<sup>-18</sup> |
-| pico | 10<sup>-15</sup> |
-| femto | 10<sup>-12</sup> |
-| nano | 10<sup>-9</sup> |
-| micro | 10<sup>-6</sup> |
-| milli | 10<sup>-3</sup> |
-| centi | 10<sup>-2</sup> |
-| deci | 10<sup>-1</sup> |
-| deca | 10<sup>1</sup> |
-| hecto | 10<sup>2</sup> |
-| kilo | 10<sup>3</sup> |
-| mega | 10<sup>6</sup> |
-| giga | 10<sup>9</sup> |
-| tera | 10<sup>12</sup> |
-| peta | 10<sup>15</sup> |
-| exa | 10<sup>18</sup> |
-
-### Usage examples
-
-| Message | Response |
-| ------- | ------ |
-| `@convert 12 celsius fahrenheit` | 12.0 celsius = 53.600054 fahrenheit |
-| `@convert 0.002 kilomile millimeter` | 0.002 kilomile = 3218688.0 millimeter |
-| `@convert 31.5 square-mile ha  | 31.5 square-mile = 8158.4625 ha |
-| `@convert 56 g lb` | 56.0 g = 0.12345887 lb |
-
-## Notes
-
-* You can use multiple `@convert` statements in a message, the response will look accordingly:
-![multiple-converts](assets/multiple-converts.png)
-
-* Enter `@convert help` to display a quick overview of the converter's functionality.
-
-* For bits and bytes, the prefixes change the figure differently: 1 kilobyte is 1024 bytes,
-1 megabyte is 1048576 bytes, etc.
--- a/api/bots/converter/test_converter.py
+++ b/api/bots/converter/test_converter.py
@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestConverterBot(BotTestCase):
-    bot_name = "converter"
-
-    def test_bot(self):
-        expected = {
-            "": ('Too few arguments given. Enter `@convert help` '
-                 'for help on using the converter.\n'),
-            "foo bar": ('Too few arguments given. Enter `@convert help` '
-                        'for help on using the converter.\n'),
-            "2 m cm": "2.0 m = 200.0 cm\n",
-            "12.0 celsius fahrenheit": "12.0 celsius = 53.600054 fahrenheit\n",
-            "0.002 kilometer millimile": "0.002 kilometer = 1.2427424 millimile\n",
-            "3 megabyte kilobit": "3.0 megabyte = 24576.0 kilobit\n",
-        }
-        self.check_expected_responses(expected)
--- a/api/bots/converter/utils.py
+++ b/api/bots/converter/utils.py
@@ -1,146 +0,0 @@
-# A dictionary allowing the conversion of each unit to its base unit.
-# An entry consists of the unit's name, a constant number and a constant
-# factor that need to be added and multiplied to convert the unit into
-# the base unit in the last parameter.
-UNITS = {'bit': [0, 1, 'bit'],
-         'byte': [0, 8, 'bit'],
-         'cubic-centimeter': [0, 0.000001, 'cubic-meter'],
-         'cubic-decimeter': [0, 0.001, 'cubic-meter'],
-         'liter': [0, 0.001, 'cubic-meter'],
-         'cubic-meter': [0, 1, 'cubic-meter'],
-         'cubic-inch': [0, 0.000016387064, 'cubic-meter'],
-         'fluid-ounce': [0, 0.000029574, 'cubic-meter'],
-         'cubic-foot': [0, 0.028316846592, 'cubic-meter'],
-         'cubic-yard': [0, 0.764554857984, 'cubic-meter'],
-         'teaspoon': [0, 0.0000049289216, 'cubic-meter'],
-         'tablespoon': [0, 0.000014787, 'cubic-meter'],
-         'cup': [0, 0.00023658823648491, 'cubic-meter'],
-         'gram': [0, 1, 'gram'],
-         'kilogram': [0, 1000, 'gram'],
-         'ton': [0, 1000000, 'gram'],
-         'ounce': [0, 28.349523125, 'gram'],
-         'pound': [0, 453.59237, 'gram'],
-         'kelvin': [0, 1, 'kelvin'],
-         'celsius': [273.15, 1, 'kelvin'],
-         'fahrenheit': [255.372222, 0.555555, 'kelvin'],
-         'centimeter': [0, 0.01, 'meter'],
-         'decimeter': [0, 0.1, 'meter'],
-         'meter': [0, 1, 'meter'],
-         'kilometer': [0, 1000, 'meter'],
-         'inch': [0, 0.0254, 'meter'],
-         'foot': [0, 0.3048, 'meter'],
-         'yard': [0, 0.9144, 'meter'],
-         'mile': [0, 1609.344, 'meter'],
-         'nautical-mile': [0, 1852, 'meter'],
-         'square-centimeter': [0, 0.0001, 'square-meter'],
-         'square-decimeter': [0, 0.01, 'square-meter'],
-         'square-meter': [0, 1, 'square-meter'],
-         'square-kilometer': [0, 1000000, 'square-meter'],
-         'square-inch': [0, 0.00064516, 'square-meter'],
-         'square-foot': [0, 0.09290304, 'square-meter'],
-         'square-yard': [0, 0.83612736, 'square-meter'],
-         'square-mile': [0, 2589988.110336, 'square-meter'],
-         'are': [0, 100, 'square-meter'],
-         'hectare': [0, 10000, 'square-meter'],
-         'acre': [0, 4046.8564224, 'square-meter']}
-
-PREFIXES = {'atto': -18,
-            'femto': -15,
-            'pico': -12,
-            'nano': -9,
-            'micro': -6,
-            'milli': -3,
-            'centi': -2,
-            'deci': -1,
-            'deca': 1,
-            'hecto': 2,
-            'kilo': 3,
-            'mega': 6,
-            'giga': 9,
-            'tera': 12,
-            'peta': 15,
-            'exa': 18}
-
-ALIASES = {'a': 'are',
-           'ac': 'acre',
-           'c': 'celsius',
-           'cm': 'centimeter',
-           'cm2': 'square-centimeter',
-           'cm3': 'cubic-centimeter',
-           'cm^2': 'square-centimeter',
-           'cm^3': 'cubic-centimeter',
-           'dm': 'decimeter',
-           'dm2': 'square-decimeter',
-           'dm3': 'cubic-decimeter',
-           'dm^2': 'square-decimeter',
-           'dm^3': 'cubic-decimeter',
-           'f': 'fahrenheit',
-           'fl-oz': 'fluid-ounce',
-           'ft': 'foot',
-           'ft2': 'square-foot',
-           'ft3': 'cubic-foot',
-           'ft^2': 'square-foot',
-           'ft^3': 'cubic-foot',
-           'g': 'gram',
-           'ha': 'hectare',
-           'in': 'inch',
-           'in2': 'square-inch',
-           'in3': 'cubic-inch',
-           'in^2': 'square-inch',
-           'in^3': 'cubic-inch',
-           'k': 'kelvin',
-           'kg': 'kilogram',
-           'km': 'kilometer',
-           'km2': 'square-kilometer',
-           'km^2': 'square-kilometer',
-           'l': 'liter',
-           'lb': 'pound',
-           'm': 'meter',
-           'm2': 'square-meter',
-           'm3': 'cubic-meter',
-           'm^2': 'square-meter',
-           'm^3': 'cubic-meter',
-           'mi': 'mile',
-           'mi2': 'square-mile',
-           'mi^2': 'square-mile',
-           'nmi': 'nautical-mile',
-           'oz': 'ounce',
-           't': 'ton',
-           'tbsp': 'tablespoon',
-           'tsp': 'teaspoon',
-           'y': 'yard',
-           'y2': 'square-yard',
-           'y3': 'cubic-yard',
-           'y^2': 'square-yard',
-           'y^3': 'cubic-yard'}
-
-HELP_MESSAGE = ('Converter usage:\n'
-                '`@convert <number> <unit_from> <unit_to>`\n'
-                'Converts `number` in the unit <unit_from> to '
-                'the <unit_to> and prints the result\n'
-                '`number`: integer or floating point number, e.g. 12, 13.05, 0.002\n'
-                '<unit_from> and <unit_to> are two of the following units:\n'
-                '* square-centimeter (cm^2, cm2), square-decimeter (dm^2, dm2), '
-                'square-meter (m^2, m2), square-kilometer (km^2, km2),'
-                ' square-inch (in^2, in2), square-foot (ft^2, ft2), square-yard (y^2, y2), '
-                ' square-mile(mi^2, mi2),  are (a), hectare (ha), acre (ac)\n'
-                '* bit, byte\n'
-                '* centimeter (cm), decimeter(dm), meter (m),'
-                ' kilometer (km), inch (in), foot (ft), yard (y),'
-                ' mile (mi), nautical-mile (nmi)\n'
-                '* Kelvin (K), Celsius(C), Fahrenheit (F)\n'
-                '* cubic-centimeter (cm^3, cm3), cubic-decimeter (dm^3, dm3), liter (l), '
-                'cubic-meter (m^3, m3), cubic-inch (in^3, in3), fluid-ounce (fl-oz), '
-                'cubic-foot (ft^3, ft3), cubic-yard (y^3, y3)\n'
-                '* gram (g), kilogram (kg), ton (t), ounce (oz), pound(lb)\n'
-                '* (metric only, U.S. and imperial units differ slightly:) teaspoon (tsp), tablespoon (tbsp), cup\n\n\n'
-                'Allowed prefixes are:\n'
-                '* atto, pico, femto, nano, micro, milli, centi, deci\n'
-                '* deca, hecto, kilo, mega, giga, tera, peta, exa\n\n\n'
-                'Usage examples:\n'
-                '* `@convert 12 celsius fahrenheit`\n'
-                '* `@convert 0.002 kilomile millimeter`\n'
-                '* `@convert 31.5 square-mile ha`\n'
-                '* `@convert 56 g lb`\n')
-
-QUICK_HELP = 'Enter `@convert help` for help on using the converter.'
--- a/api/bots/define/assets/correct_word.png
+++ b/api/bots/define/assets/correct_word.png
--- a/api/bots/define/assets/wrong_word.png
+++ b/api/bots/define/assets/wrong_word.png
--- a/api/bots/define/define.py
+++ b/api/bots/define/define.py
@@ -1,66 +0,0 @@
-# See readme.md for instructions on running this code.
-import logging
-import json
-import requests
-import html2text
-
-class DefineHandler(object):
-    '''
-    This plugin define a word that the user inputs. It
-    looks for messages starting with '@mention-bot'.
-    '''
-
-    DEFINITION_API_URL = 'https://owlbot.info/api/v1/dictionary/{}?format=json'
-    REQUEST_ERROR_MESSAGE = 'Definition not available.'
-    EMPTY_WORD_REQUEST_ERROR_MESSAGE = 'Please enter a word to define.'
-    PHRASE_ERROR_MESSAGE = 'Definitions for phrases are not available.'
-
-    def usage(self):
-        return '''
-            This plugin will allow users to define a word. Users should preface
-            messages with @mention-bot.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        original_content = message['content'].strip()
-        bot_response = self.get_bot_define_response(original_content)
-
-        client.send_reply(message, bot_response)
-
-    def get_bot_define_response(self, original_content):
-        split_content = original_content.split(' ')
-        # If there are more than one word (a phrase)
-        if len(split_content) > 1:
-            return DefineHandler.PHRASE_ERROR_MESSAGE
-
-        to_define = split_content[0].strip()
-        to_define_lower = to_define.lower()
-
-        # No word was entered.
-        if not to_define_lower:
-            return self.EMPTY_WORD_REQUEST_ERROR_MESSAGE
-        else:
-            response = '**{}**:\n'.format(to_define)
-
-            try:
-                # Use OwlBot API to fetch definition.
-                api_result = requests.get(self.DEFINITION_API_URL.format(to_define_lower))
-                # Convert API result from string to JSON format.
-                definitions = api_result.json()
-
-                # Could not fetch definitions for the given word.
-                if not definitions:
-                    response += self.REQUEST_ERROR_MESSAGE
-                else:  # Definitions available.
-                    # Show definitions line by line.
-                    for d in definitions:
-                        example = d['example'] if d['example'] else '*No example available.*'
-                        response += '\n' + '* (**{}**) {}\n&nbsp;&nbsp;{}'.format(d['type'], d['defenition'], html2text.html2text(example))
-
-            except Exception as e:
-                response += self.REQUEST_ERROR_MESSAGE
-                logging.exception(e)
-
-            return response
-
-handler_class = DefineHandler
--- a/api/bots/define/readme.md
+++ b/api/bots/define/readme.md
@@ -1,21 +0,0 @@
-# DefineBot
-
-* This is a bot that defines a word that the user inputs. Whenever the user
-inputs a message starting with '@define', the bot defines the word
-that follows.
-
-* The definitions are brought to the website using an API. The bot posts the
-definition of the word to the stream from which the user inputs the message.
-If the user inputs a word that does not exist or a word that is incorrect or
-is not in the dictionary, the definition is not displayed.
-
-* For example, if the user says "@define crash", all the meanings of crash
-appear, each in a separate line.
-
-![Correct Word](assets/correct_word.png)
-
-* If the user enters a wrong word, like "@define cresh" or "@define crish",
-then an error message saying no definition is available is displayed.
-
-![Wrong Word](assets/wrong_word.png)
-
--- a/api/bots/define/test_define.py
+++ b/api/bots/define/test_define.py
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestDefineBot(BotTestCase):
-    bot_name = "define"
-
-    def test_bot(self):
-        expected = {
-            "": 'Please enter a word to define.',
-            "foo": "**foo**:\nDefinition not available.",
-            "cat": ("**cat**:\n\n* (**noun**) a small domesticated carnivorous mammal "
-                    "with soft fur, a short snout, and retractile claws. It is widely "
-                    "kept as a pet or for catching mice, and many breeds have been "
-                    "developed.\n&nbsp;&nbsp;their pet cat\n\n"),
-        }
-        self.check_expected_responses(expected)
--- a/api/bots/encrypt/assets/EncryptBot-terminal.png
+++ b/api/bots/encrypt/assets/EncryptBot-terminal.png
--- a/api/bots/encrypt/assets/EncryptBot-test.png
+++ b/api/bots/encrypt/assets/EncryptBot-test.png
--- a/api/bots/encrypt/assets/EncryptBot-test2.png
+++ b/api/bots/encrypt/assets/EncryptBot-test2.png
--- a/api/bots/encrypt/assets/EncryptBot-test3.png
+++ b/api/bots/encrypt/assets/EncryptBot-test3.png
--- a/api/bots/encrypt/encrypt.py
+++ b/api/bots/encrypt/encrypt.py
@@ -1,45 +0,0 @@
-def encrypt(text):
-    # This is where the actual ROT13 is applied
-    # WHY IS .JOIN NOT WORKING?!
-    textlist = list(text)
-    newtext = ''
-    firsthalf = 'abcdefghijklmABCDEFGHIJKLM'
-    lasthalf = 'nopqrstuvwxyzNOPQRSTUVWXYZ'
-    for char in textlist:
-        if char in firsthalf:
-            newtext += lasthalf[firsthalf.index(char)]
-        elif char in lasthalf:
-            newtext += firsthalf[lasthalf.index(char)]
-        else:
-            newtext += char
-
-    return newtext
-
-class EncryptHandler(object):
-    '''
-    This bot allows users to quickly encrypt messages using ROT13 encryption.
-    It encrypts/decrypts messages starting with @mention-bot.
-    '''
-
-    def usage(self):
-        return '''
-            This bot uses ROT13 encryption for its purposes.
-            It responds to me starting with @mention-bot.
-            Feeding encrypted messages into the bot decrypts them.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        bot_response = self.get_bot_encrypt_response(message)
-        client.send_reply(message, bot_response)
-
-    def get_bot_encrypt_response(self, message):
-        original_content = message['content']
-        temp_content = encrypt(original_content)
-        send_content = "Encrypted/Decrypted text: " + temp_content
-        return send_content
-
-handler_class = EncryptHandler
-
-if __name__ == '__main__':
-    assert encrypt('ABCDabcd1234') == 'NOPQnopq1234'
-    assert encrypt('NOPQnopq1234') == 'ABCDabcd1234'
--- a/api/bots/encrypt/readme.md
+++ b/api/bots/encrypt/readme.md
@@ -1,16 +0,0 @@
-About EncryptBot:
-EncryptBot Allows for quick ROT13 encryption in the middle of a chat.
-
-What It Does:
-The bot encrypts any message sent to it on any stream it is subscribed to with ROT13.
-
-How It Works:
-The bot will Use ROT13(A -> N, B -> O... and vice-versa) in a python
-implementation to provide quick and easy encryption.
-
-How to Use:
-
-Send the message you want to encrypt, add @encrypt to the beginning.
-The Encrypted message will be sent back to the stream the original
-message was posted in to the topic <sender-email>'s encrypted text.
-Messages can be decrypted by sending them to EncryptBot in the same way.
--- a/api/bots/encrypt/test_encrypt.py
+++ b/api/bots/encrypt/test_encrypt.py
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestEncryptBot(BotTestCase):
-    bot_name = "encrypt"
-
-    def test_bot(self):
-        expected = {
-            "": "Encrypted/Decrypted text: ",
-            "Let\'s Do It": "Encrypted/Decrypted text: Yrg\'f Qb Vg",
-            "me&mom together..!!": "Encrypted/Decrypted text: zr&zbz gbtrgure..!!",
-            "foo bar": "Encrypted/Decrypted text: sbb one",
-            "Please encrypt this": "Encrypted/Decrypted text: Cyrnfr rapelcg guvf",
-        }
-        self.check_expected_responses(expected)
--- a/api/bots/followup/followup.py
+++ b/api/bots/followup/followup.py
@@ -1,46 +0,0 @@
-# See readme.md for instructions on running this code.
-
-class FollowupHandler(object):
-    '''
-    This plugin facilitates creating follow-up tasks when
-    you are using Zulip to conduct a virtual meeting.  It
-    looks for messages starting with '@mention-bot'.
-
-    In this example, we write follow up items to a special
-    Zulip stream called "followup," but this code could
-    be adapted to write follow up items to some kind of
-    external issue tracker as well.
-    '''
-
-    def usage(self):
-        return '''
-            This plugin will allow users to flag messages
-            as being follow-up items.  Users should preface
-            messages with "@mention-bot".
-
-            Before running this, make sure to create a stream
-            called "followup" that your API user can send to.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        if message['content'] == '':
-            bot_response = "Please specify the message you want to send to followup stream after @mention-bot"
-            client.send_reply(message, bot_response)
-        else:
-            bot_response = self.get_bot_followup_response(message)
-            client.send_message(dict(
-                type='stream',
-                to='followup',
-                subject=message['sender_email'],
-                content=bot_response,
-            ))
-
-    def get_bot_followup_response(self, message):
-        original_content = message['content']
-        original_sender = message['sender_email']
-        temp_content = 'from %s: ' % (original_sender,)
-        new_content = temp_content + original_content
-
-        return new_content
-
-handler_class = FollowupHandler
--- a/api/bots/followup/test_followup.py
+++ b/api/bots/followup/test_followup.py
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestFollowUpBot(BotTestCase):
-    bot_name = "followup"
-
-    def test_bot(self):
-        expected_send_reply = {
-            "": 'Please specify the message you want to send to followup stream after @mention-bot'
-        }
-        self.check_expected_responses(expected_send_reply, expected_method='send_reply')
-
-        expected_send_message = {
-            "foo": {
-                'type': 'stream',
-                'to': 'followup',
-                'subject': 'foo_sender@zulip.com',
-                'content': 'from foo_sender@zulip.com: foo',
-            },
-            "I have completed my task": {
-                'type': 'stream',
-                'to': 'followup',
-                'subject': 'foo_sender@zulip.com',
-                'content': 'from foo_sender@zulip.com: I have completed my task',
-            },
-        }
-        self.check_expected_responses(expected_send_message, expected_method='send_message')
--- a/api/bots/foursquare/foursquare.config
+++ b/api/bots/foursquare/foursquare.config
@@ -1,2 +0,0 @@
-[Foursquare]
-api_key = abcdefghijksm
--- a/api/bots/foursquare/foursquare.py
+++ b/api/bots/foursquare/foursquare.py
@@ -1,123 +0,0 @@
-from __future__ import print_function
-from __future__ import absolute_import
-
-import datetime as dt
-import re
-import requests
-from os.path import expanduser
-from six.moves import configparser as cp
-from six.moves import range
-
-home = expanduser('~')
-CONFIG_PATH = home + '/zulip/api/bots/foursquare/foursquare.config'
-
-def get_api_key():
-    # foursquare.config must have been moved from
-    # ~/zulip/api/bots/foursquare/foursquare.config into
-    # ~/foursquare.config for program to work
-    # see readme.md for more information
-    with open(CONFIG_PATH) as settings:
-        config = cp.ConfigParser()
-        config.readfp(settings)
-        return config.get('Foursquare', 'api_key')
-
-class FoursquareHandler(object):
-    def __init__(self):
-        self.api_key = get_api_key()
-
-    def usage(self):
-        return '''
-            This plugin allows users to search for restaurants nearby an inputted
-            location to a limit of 3 venues for every location. The name, address
-            and description of the restaurant will be outputted.
-            It looks for messages starting with '@mention-bot'.
-            If you need help, simply type:
-            @mention-bot /help into the Compose Message box
-
-            Sample input:
-            @mention-bot Chicago, IL
-            @mention-bot help
-            '''
-
-    help_info = '''
-The Foursquare bot can receive keyword limiters that specify the location, distance (meters) and
-cusine of a restaurant in that exact order.
-Please note the required use of quotes in the search location.
-
-Example Inputs:
-@mention-bot 'Millenium Park' 8000 donuts
-@mention-bot 'Melbourne, Australia' 40000 seafood
-                '''
-
-    def format_json(self, venues):
-        def format_venue(venue):
-            name = venue['name']
-            address = ', '.join(venue['location']['formattedAddress'])
-            keyword = venue['categories'][0]['pluralName']
-            blurb = '\n'.join([name, address, keyword])
-            return blurb
-
-        return '\n'.join(format_venue(venue) for venue in venues)
-
-    def send_info(self, message, letter, client):
-        client.send_reply(message, letter)
-
-    def handle_message(self, message, client, state_handler):
-        words = message['content'].split()
-        if "/help" in words:
-            self.send_info(message, self.help_info, client)
-            return
-
-        # These are required inputs for the HTTP request.
-        try:
-            params = {'limit': '3'}
-            params['near']  = re.search('\'[A-Za-z]\w+[,]?[\s\w+]+?\'', message['content']).group(0)
-            params['v'] = 20170108
-            params['oauth_token'] = self.api_key
-        except AttributeError:
-            pass
-
-        # Optional params for HTTP request.
-        if len(words) >= 1:
-            try:
-                params['radius'] = re.search('([0-9]){3,}', message['content']).group(0)
-            except AttributeError:
-                pass
-            try:
-                params['query'] = re.search('\s([A-Za-z]+)$', message['content']).group(0)[1:]
-            except AttributeError:
-                params['query'] = 'food'
-
-        response = requests.get('https://api.foursquare.com/v2/venues/search?',
-                                params=params)
-        print(response.url)
-        if response.status_code == 200:
-            received_json = response.json()
-        else:
-            self.send_info(message,
-                           "Invalid Request\nIf stuck, try '@mention-bot help'.",
-                           client)
-            return
-
-        if received_json['meta']['code'] == 200:
-            response_msg = ('Food nearby ' + params['near'] +
-                            ' coming right up:\n' +
-                            self.format_json(received_json['response']['venues']))
-            self.send_info(message, response_msg, client)
-            return
-
-        self.send_info(message,
-                       "Invalid Request\nIf stuck, try '@mention-bot help'.",
-                       client)
-        return
-
-handler_class = FoursquareHandler
-
-def test_get_api_key():
-    # must change to your own api key for test to work
-    result = get_api_key()
-    assert result == 'abcdefghijksm'
-
-if __name__ == '__main__':
-    test_get_api_key()
-    print('Success')
--- a/api/bots/foursquare/readme.md
+++ b/api/bots/foursquare/readme.md
@@ -1,32 +0,0 @@
-# FourSquare Bot
-
-* This is a bot that returns a list of restaurants from a user input of location,
-proximity and restaurant type in that exact order. The number of returned
-restaurants are capped at 3 per request.
-
-* The list of restaurants are brought to Zulip using an API. The bot sends a GET
-request to https://api.foursquare.com/v2/. If the user does not correctly input
-a location, proximity and a restaurant type, the bot will return an error message.
-
-* For example, if the user says "@foursquare 'Chicago, IL' 80000 seafood", the bot
-will return:
-
-Food nearby 'Chicago, IL' coming right up:
-
-    Dee's Seafood Co.
-    2723 S Poplar Ave, Chicago, IL 60608, United States
-    Fish Markets
-
-    Seafood Harbor
-    2131 S Archer Ave (at Wentworth Ave), Chicago, IL 60616, United States
-    Seafood Restaurants
-
-    Joe's Seafood, Prime Steak & Stone Crab
-    60 E Grand Ave (at N Rush St), Chicago, IL 60611, United States
-    Seafood Restaurants
-
-* If the user enters a wrong word, like "@foursquare 80000 donuts" or "@foursquare",
-then an error message saying invalid input will be displayed.
-
-* To get the required API key, visit: https://developer.foursquare.com/overview/auth
-for more information.
--- a/api/bots/giphy/giphy.py
+++ b/api/bots/giphy/giphy.py
@@ -1,93 +0,0 @@
-# To use this plugin, you need to set up the Giphy API key for this bot in
-# ~/.giphy_config
-
-from __future__ import absolute_import
-from __future__ import print_function
-from six.moves.configparser import SafeConfigParser
-import requests
-import logging
-import sys
-import os
-import re
-
-GIPHY_TRANSLATE_API = 'http://api.giphy.com/v1/gifs/translate'
-
-if not os.path.exists(os.environ['HOME'] + '/.giphy_config'):
-    print('Giphy bot config file not found, please set up it in ~/.giphy_config'
-          '\n\nUsing format:\n\n[giphy-config]\nkey=<giphy API key here>\n\n')
-    sys.exit(1)
-
-
-class GiphyHandler(object):
-    '''
-    This plugin posts a GIF in response to the keywords provided by the user.
-    Images are provided by Giphy, through the public API.
-    The bot looks for messages starting with @mention of the bot
-    and responds with a message with the GIF based on provided keywords.
-    It also responds to private messages.
-    '''
-    def usage(self):
-        return '''
-            This plugin allows users to post GIFs provided by Giphy.
-            Users should preface keywords with the Giphy-bot @mention.
-            The bot responds also to private messages.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        bot_response = get_bot_giphy_response(message, client)
-        client.send_reply(message, bot_response)
-
-
-class GiphyNoResultException(Exception):
-    pass
-
-
-def get_giphy_api_key_from_config():
-    config = SafeConfigParser()
-    with open(os.environ['HOME'] + '/.giphy_config', 'r') as config_file:
-        config.readfp(config_file)
-    return config.get("giphy-config", "key")
-
-
-def get_url_gif_giphy(keyword, api_key):
-    # Return a URL for a Giphy GIF based on keywords given.
-    # In case of error, e.g. failure to fetch a GIF URL, it will
-    # return a number.
-    query = {'s': keyword,
-             'api_key': api_key}
-    try:
-        data = requests.get(GIPHY_TRANSLATE_API, params=query)
-    except requests.exceptions.ConnectionError as e:  # Usually triggered by bad connection.
-        logging.warning(e)
-        raise
-
-    search_status = data.json()['meta']['status']
-    if search_status != 200 or not data.ok:
-        raise requests.exceptions.ConnectionError
-
-    try:
-        gif_url = data.json()['data']['images']['original']['url']
-    except (TypeError, KeyError):  # Usually triggered by no result in Giphy.
-        raise GiphyNoResultException()
-
-    return gif_url
-
-
-def get_bot_giphy_response(message, client):
-    # Each exception has a specific reply should "gif_url" return a number.
-    # The bot will post the appropriate message for the error.
-    keyword = message['content']
-    try:
-        gif_url = get_url_gif_giphy(keyword, get_giphy_api_key_from_config())
-    except requests.exceptions.ConnectionError:
-        return ('Uh oh, sorry :slightly_frowning_face:, I '
-                'cannot process your request right now. But, '
-                'let\'s try again later! :grin:')
-    except GiphyNoResultException:
-        return ('Sorry, I don\'t have a GIF for "%s"! '
-                ':astonished:' % (keyword))
-    return ('[Click to enlarge](%s)'
-            '[](/static/images/interactive-bot/giphy/powered-by-giphy.png)'
-            % (gif_url))
-
-handler_class = GiphyHandler
--- a/api/bots/giphy/test_giphy.py
+++ b/api/bots/giphy/test_giphy.py
@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-import json
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-from bots.giphy import giphy
-
-def get_http_response_json(gif_url):
-    response_json = {
-        'meta': {
-            'status': 200
-        },
-        'data': {
-            'images': {
-                'original': {
-                    'url': gif_url
-                }
-            }
-        }
-    }
-    return response_json
-
-def get_bot_response(gif_url):
-    return ('[Click to enlarge](%s)'
-            '[](/static/images/interactive-bot/giphy/powered-by-giphy.png)'
-            % (gif_url))
-
-def get_http_request(keyword):
-    return {
-        'api_url': giphy.GIPHY_TRANSLATE_API,
-        'params': {
-            's': keyword,
-            'api_key': giphy.get_giphy_api_key_from_config()
-        }
-    }
-
-class TestGiphyBot(BotTestCase):
-    bot_name = "giphy"
-
-    def test_bot(self):
-        # This message calls `send_reply` function of BotHandlerApi
-        keyword = "Hello"
-        gif_url = "https://media4.giphy.com/media/3o6ZtpxSZbQRRnwCKQ/giphy.gif"
-        expectations = {
-            keyword: get_bot_response(gif_url)
-        }
-        self.check_expected_responses(
-            expectations=expectations,
-            http_request=get_http_request(keyword),
-            http_response=get_http_response_json(gif_url)
-        )
--- a/api/bots/git_hub_comment/init.py
+++ b/api/bots/git_hub_comment/init.py
--- a/api/bots/git_hub_comment/git_hub_comment.py
+++ b/api/bots/git_hub_comment/git_hub_comment.py
@@ -1,126 +0,0 @@
-# See readme.md for instructions on running this code.
-from __future__ import absolute_import
-from __future__ import print_function
-from . import github
-import json
-import logging
-import os
-import requests
-
-
-class InputError(IndexError):
-    '''raise this when there is an error with the information the user has entered'''
-
-
-class GitHubHandler(object):
-    '''
-    This plugin allows you to comment on a GitHub issue, under a certain repository.
-    It looks for messages starting with '@mention-bot'.
-    '''
-
-    def usage(self):
-        return '''
-            This bot will allow users to comment on a GitHub issue.
-            Users should preface messages with '@mention-bot'.
-            You will need to have a GitHub account.
-
-            Before running this, make sure to get a GitHub OAuth token.
-            The token will need to be authorized for the following scopes:
-            'gist, public_repo, user'.
-            Store it in the '~/.github_auth.conf' file, along with your username, in the format:
-            github_repo = <repo_name>  (The name of the repo to post to)
-            github_repo_owner = <repo_owner>  (The owner of the repo to post to)
-            github_username = <username>  (The username of the GitHub bot)
-            github_token = <oauth_token>   (The personal access token for the GitHub bot)
-
-            Leave the first two options blank.
-
-            Please use this format in your message to the bot:
-            '<repository_owner>/<repository>/<issue_number>/<your_comment>'.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        original_content = message['content']
-        original_sender = message['sender_email']
-
-        handle_input(client, original_content, original_sender)
-
-handler_class = GitHubHandler
-
-
-def send_to_github(repo_owner, repo, issue, comment_body):
-    session = github.auth()
-    comment = {
-        'body': comment_body
-    }
-    r = session.post('https://api.github.com/repos/%s/%s/issues/%s/comments' % (repo_owner, repo, issue),
-                     json.dumps(comment))
-
-    return r.status_code
-
-
-def get_values_message(original_content):
-    # gets rid of whitespace around the edges, so that they aren't a problem in the future
-    message_content = original_content.strip()
-    # splits message by '/' which will work if the information was entered correctly
-    message_content = message_content.split('/')
-    try:
-        # this will work if the information was entered correctly
-        user = github.get_username()
-        repo_owner = message_content[2]
-        repo = message_content[3]
-        issue = message_content[4]
-        comment_body = message_content[5]
-
-        return dict(user=user, repo_owner=repo_owner, repo=repo, issue=issue, comment_body=comment_body)
-    except IndexError:
-        raise InputError
-
-
-def handle_input(client, original_content, original_sender):
-    try:
-        params = get_values_message(original_content)
-
-        status_code = send_to_github(params['repo_owner'], params['repo'],
-                                     params['issue'], params['comment_body'])
-
-        if status_code == 201:
-            # sending info to github was successful!
-            reply_message = "You commented on issue number " + params['issue'] + " under " + \
-                            params['repo_owner'] + "'s repository " + params['repo'] + "!"
-
-            send_message(client, reply_message, original_sender)
-
-        elif status_code == 404:
-            # this error could be from an error with the OAuth token
-            reply_message = "Error code: " + str(status_code) + " :( There was a problem commenting on issue number " \
-                            + params['issue'] + " under " + \
-                            params['repo_owner'] + "'s repository " + params['repo'] + \
-                            ". Do you have the right OAuth token?"
-
-            send_message(client, reply_message, original_sender)
-
-        else:
-            # sending info to github did not work
-            reply_message = "Error code: " + str(status_code) +\
-                            " :( There was a problem commenting on issue number " \
-                            + params['issue'] + " under " + \
-                            params['repo_owner'] + "'s repository " + params['repo'] + \
-                            ". Did you enter the information in the correct format?"
-
-            send_message(client, reply_message, original_sender)
-    except InputError:
-            message = "It doesn't look like the information was entered in the correct format." \
-                      " Did you input it like this? " \
-                      "'/<username>/<repository_owner>/<repository>/<issue_number>/<your_comment>'."
-            send_message(client, message, original_sender)
-            logging.error('there was an error with the information you entered')
-
-
-def send_message(client, message, original_sender):
-    # function for sending a message
-    client.send_message(dict(
-        type='private',
-        to=original_sender,
-        content=message,
-    ))
--- a/api/bots/git_hub_comment/readme.md
+++ b/api/bots/git_hub_comment/readme.md
@@ -1,53 +0,0 @@
-# Overview
-
-This is the documentation for how to set up and run the GitHub comment bot. (`git_hub_comment.py`)
-
-This directory contains library code for running Zulip
-bots that react to messages sent by users.
-
-This bot will allow you to comment on a GitHub issue.
-You should preface messages with `@comment` or `@gcomment`.
-You will need to have a GitHub account, and a GitHub OAuth token.
-
-## Setup
-Before running this bot, make sure to get a GitHub OAuth token.
-You can look at this tutorial if you need help:
-<https://help.github.com/articles/creating-an-access-token-for-command-line-use/>
-The token will need to be authorized for the following scopes: `gist, public_repo, user`.
-Store it in the `~/github-auth.conf` file, along with your username, in the format:
-github_repo = <repo_name>  (The name of the repo to post to)
-github_repo_owner = <repo_owner>  (The owner of the repo to post to)
-github_username = <username>  (The username of the GitHub bot)
-github_token = <oauth_token>   (The personal access token for the GitHub bot)
-`<repository_owner>/<repository>/<issue_number>/<your_comment`.
-
-## Running the bot
-
-Here is an example of running the `git_hub_comment` bot from
-inside a Zulip repo:
-
-    `cd ~/zulip/api`
-    `bots_api/run.py bots/git_hub_comment/git_hub_comment.py --config-file ~/.zuliprc-prod`
-
-Once the bot code starts running, you will see a
-message explaining how to use the bot, as well as
-some log messages.  You can use the `--quiet` option
-to suppress some of the informational messages.
-
-The bot code will run continuously until you kill them with
-control-C (or otherwise).
-
-### Configuration
-
-For this document we assume you have some prior experience
-with using the Zulip API, but here is a quick review of
-what a `.zuliprc` files looks like.  You can connect to the
-API as your own human user, or you can go into the Zulip settings
-page to create a user-owned bot.
-
-    [api]
-    email=someuser@example.com
-    key=<your api key>
-    site=https://zulip.somewhere.com
-
-
--- a/api/bots/github/init.py
+++ b/api/bots/github/init.py
--- a/api/bots/github/github.py
+++ b/api/bots/github/github.py
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-
-# The purpose of this file is to handle all requests
-# to the Github API, which will make sure that all
-# requests are to the same account, and that all requests
-# authenticated correctly and securely
-# The sole purpose of this is to authenticate, and it will
-# return the requests session that is properly authenticated
-import logging
-import os
-import requests
-import six.moves.configparser
-
-
-# This file contains the oauth token for a github user, their username, and optionally, a repository
-# name and owner. All in the format:
-# github_repo
-# github_repo_owner
-# github_username
-# github_token
-CONFIG_FILE = '~/.github-auth.conf'
-
-global config
-config = six.moves.configparser.ConfigParser()  # Sets up the configParser to read the .conf file
-config.read([os.path.expanduser(CONFIG_FILE)])  # Reads the config file
-
-
-def auth():
-    # Creates and authorises a requests session
-    session = requests.session()
-    session.auth = (get_username(), get_oauth_token())
-
-    return session
-
-
-def get_oauth_token():
-    _get_data('token')
-
-
-def get_username():
-    _get_data('username')
-
-
-def get_repo():
-    _get_data('repo')
-
-
-def get_repo_owner():
-    _get_data('repo_owner')
-
-
-def _get_data(key):
-    try:
-        return config.get('github', 'github_%s' % (key))
-    except Exception:
-        logging.exception('GitHub %s not supplied in ~/.github-auth.conf.' % (key))
--- a/api/bots/github_issues/init.py
+++ b/api/bots/github_issues/init.py
--- a/api/bots/github_issues/github_issues.py
+++ b/api/bots/github_issues/github_issues.py
@@ -1,109 +0,0 @@
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from . import github
-import json
-import os
-import requests
-import six.moves.configparser
-import urllib.error
-import urllib.parse
-import urllib.request
-
-class IssueHandler(object):
-    '''
-    This plugin facilitates sending issues to github, when
-    an item is prefixed with '@mention-bot'.
-
-    It will also write items to the issues stream, as well
-    as reporting it to github
-    '''
-
-    URL = 'https://api.github.com/repos/{}/{}/issues'
-    CHARACTER_LIMIT = 70
-    CONFIG_FILE = '~/.github-auth.conf'
-
-    def __init__(self):
-        self.repo_name = github.get_repo()
-        self.repo_owner = github.get_repo_owner()
-
-    def usage(self):
-        return '''
-            This plugin will allow users to flag messages
-            as being issues with Zulip by using te prefix '@mention-bot'.
-
-            Before running this, make sure to create a stream
-            called "issues" that your API user can send to.
-
-            Also, make sure that the credentials of the github bot have
-            been typed in correctly, that there is a personal access token
-            with access to public repositories ONLY,
-            and that the repository name is entered correctly.
-
-            Check ~/.github-auth.conf, and make sure there are
-            github_repo = <repo_name>  (The name of the repo to post to)
-            github_repo_owner = <repo_owner>  (The owner of the repo to post to)
-            github_username = <username>  (The username of the GitHub bot)
-            github_token = <oauth_token>   (The personal access token for the GitHub bot)
-            '''
-
-    def handle_message(self, message, client, state_handler):
-
-        original_content = message['content']
-        original_sender = message['sender_email']
-
-        temp_content = 'by {}:'.format(original_sender,)
-        new_content = temp_content + original_content
-        # gets the repo url
-        url_new = self.URL.format(self.REPO_OWNER, self.REPO_NAME)
-
-        # signs into github using the provided username and password
-        session = github.auth()
-
-        issue_title = message['content'].strip()
-        issue_content = ''
-        new_issue_title = ''
-        for part_of_title in issue_title.split():
-            if len(new_issue_title) < self.CHARACTER_LIMIT:
-                new_issue_title += '{} '.format(part_of_title)
-            else:
-                issue_content += '{} '.format(part_of_title)
-
-        new_issue_title = new_issue_title.strip()
-        issue_content = issue_content.strip()
-        new_issue_title += '...'
-
-        # Creates the issue json, that is transmitted to the github api servers
-        issue = {
-            'title': new_issue_title,
-            'body': '{} **Sent by [{}](https://chat.zulip.org/#) from zulip**'.format(issue_content, original_sender),
-            'assignee': '',
-            'milestone': 'none',
-            'labels': [''],
-        }
-        # Sends the HTTP post request
-        r = session.post(url_new, json.dumps(issue))
-
-        if r.ok:
-            # sends the message onto the 'issues' stream so it can be seen by zulip users
-            client.send_message(dict(
-                type='stream',
-                to='issues',
-                subject=message['sender_email'],
-                # Adds a check mark so that the user can verify if it has been sent
-                content='{} :heavy_check_mark:'.format(new_content),
-            ))
-            return
-            # This means that the issue has not been sent
-            # sends the message onto the 'issues' stream so it can be seen by zulip users
-            client.send_message(dict(
-                type='stream',
-                to='issues',
-                subject=message['sender_email'],
-                # Adds a cross so that the user can see that it has failed, and provides a link to a
-                # google search that can (hopefully) direct them to the error
-                content='{} :x: Code: [{}](https://www.google.com/search?q=Github HTTP {} Error {})'
-                        .format(new_content, r.status_code, r.status_code, r.content),
-            ))
-
-handler_class = IssueHandler
--- a/api/bots/googlesearch/init.py
+++ b/api/bots/googlesearch/init.py
--- a/api/bots/googlesearch/googlesearch.py
+++ b/api/bots/googlesearch/googlesearch.py
@@ -1,93 +0,0 @@
-# See readme.md for instructions on running this code.
-from __future__ import print_function
-import logging
-import http.client
-from six.moves.urllib.request import urlopen
-
-# Uses the Google search engine bindings
-#   pip install --upgrade google
-from google import search
-
-
-def get_google_result(search_keywords):
-    help_message = "To use this bot, start messages with @mentioned-bot, \
-                    followed by what you want to search for. If \
-                    found, Zulip will return the first search result \
-                    on Google.\
-                    \
-                    An example message that could be sent is:\
-                    '@mentioned-bot zulip' or \
-                    '@mentioned-bot how to create a chatbot'."
-    if search_keywords == 'help':
-        return help_message
-    elif search_keywords == '' or search_keywords is None:
-        return help_message
-    else:
-        try:
-            urls = search(search_keywords, stop=20)
-            urlopen('http://216.58.192.142', timeout=1)
-        except http.client.RemoteDisconnected as er:
-            logging.exception(er)
-            return 'Error: No internet connection. {}.'.format(er)
-        except Exception as e:
-            logging.exception(e)
-            return 'Error: Search failed. {}.'.format(e)
-
-        try:
-            url = next(urls)
-        except AttributeError as a_err:
-            # google.search query failed and urls is of object
-            # 'NoneType'
-            logging.exception(a_err)
-            return "Error: Google search failed with a NoneType result. {}.".format(a_err)
-        except TypeError as t_err:
-            # google.search query failed and returned None
-            # This technically should not happen but the prior
-            # error check assumed this behavior
-            logging.exception(t_err)
-            return "Error: Google search function failed. {}.".format(t_err)
-        except Exception as e:
-            logging.exception(e)
-            return 'Error: Search failed. {}.'.format(e)
-
-        return 'Success: {}'.format(url)
-
-
-class GoogleSearchHandler(object):
-    '''
-    This plugin allows users to enter a search
-    term in Zulip and get the top URL sent back
-    to the context (stream or private) in which
-    it was called. It looks for messages starting
-    with @mentioned-bot.
-    '''
-
-    def usage(self):
-        return '''
-            This plugin will allow users to search
-            for a given search term on Google from
-            Zulip. Use '@mentioned-bot help' to get
-            more information on the bot usage. Users
-            should preface messages with
-            @mentioned-bot.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        original_content = message['content']
-        result = get_google_result(original_content)
-        client.send_reply(message, result)
-
-handler_class = GoogleSearchHandler
-
-
-def test():
-    try:
-        urlopen('http://216.58.192.142', timeout=1)
-        print('Success')
-        return True
-    except http.client.RemoteDisconnected as e:
-        print('Error: {}'.format(e))
-        return False
-
-if __name__ == '__main__':
-    test()
--- a/api/bots/googlesearch/readme.md
+++ b/api/bots/googlesearch/readme.md
@@ -1,23 +0,0 @@
-# Google Search bot
-
-This bot allows users to do Google search queries and have the bot
-respond with the first search result.  It is by default set to the
-highest safe-search setting.
-
-## Usage
-
-Run this bot as described
-[here](http://zulip.readthedocs.io/en/latest/bots-guide.html#how-to-deploy-a-bot).
-
-Use this bot with the following command
-
-`@mentioned-bot <search terms>`
-
-This will return the first link found by Google for `<search terms>`
-and print the resulting URL.
-
-If no `<search terms>` are entered, a help message is printed instead.
-
-If there was an error in the process of running the search (socket
-errors, Google search function failed, or general failures), an error
-message is returned.
--- a/api/bots/helloworld/init.py
+++ b/api/bots/helloworld/init.py
--- a/api/bots/helloworld/helloworld.py
+++ b/api/bots/helloworld/helloworld.py
@@ -1,18 +0,0 @@
-# See readme.md for instructions on running this code.
-
-
-class HelloWorldHandler(object):
-    def usage(self):
-        return '''
-        This is a boilerplate bot that responds to a user query with
-        "beep boop", which is robot for "Hello World".
-
-        This bot can be used as a template for other, more
-        sophisticated, bots.
-        '''
-
-    def handle_message(self, message, client, state_handler):
-        content = 'beep boop'
-        client.send_reply(message, content)
-
-handler_class = HelloWorldHandler
--- a/api/bots/helloworld/readme.md
+++ b/api/bots/helloworld/readme.md
@@ -1,4 +0,0 @@
-Simple Zulip bot that will respond to any query with a "beep boop".
-
-The helloworld bot is a boilerplate bot that can be used as a
-template for more sophisticated/evolved Zulip bots.
--- a/api/bots/helloworld/test_helloworld.py
+++ b/api/bots/helloworld/test_helloworld.py
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-from six.moves import zip
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestHelloWorldBot(BotTestCase):
-    bot_name = "helloworld"
-
-    def test_bot(self):
-        txt = "beep boop"
-        messages = ["", "foo", "Hi, my name is abc"]
-        self.check_expected_responses(dict(list(zip(messages, len(messages)*[txt]))))
--- a/api/bots/help/init.py
+++ b/api/bots/help/init.py
--- a/api/bots/help/help.py
+++ b/api/bots/help/help.py
@@ -1,18 +0,0 @@
-# See readme.md for instructions on running this code.
-
-class HelpHandler(object):
-    def usage(self):
-        return '''
-            This plugin will give info about Zulip to
-            any user that types a message saying "help".
-
-            This is example code; ideally, you would flesh
-            this out for more useful help pertaining to
-            your Zulip instance.
-            '''
-
-    def handle_message(self, message, client, state_handler):
-        help_content = "Info on Zulip can be found here:\nhttps://github.com/zulip/zulip"
-        client.send_reply(message, help_content)
-
-handler_class = HelpHandler
--- a/api/bots/help/test_help.py
+++ b/api/bots/help/test_help.py
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import absolute_import
-from __future__ import print_function
-
-import os
-import sys
-from six.moves import zip
-
-our_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, os.path.normpath(os.path.join(our_dir)))
-# For dev setups, we can find the API in the repo itself.
-if os.path.exists(os.path.join(our_dir, '..')):
-    sys.path.insert(0, '..')
-from bots_test_lib import BotTestCase
-
-class TestHelpBot(BotTestCase):
-    bot_name = "help"
-
-    def test_bot(self):
-        txt = "Info on Zulip can be found here:\nhttps://github.com/zulip/zulip"
-        messages = ["", "help", "Hi, my name is abc"]
-        self.check_expected_responses(dict(list(zip(messages, len(messages)*[txt]))))
--- a/api/bots/howdoi/init.py
+++ b/api/bots/howdoi/init.py
--- a/api/bots/howdoi/assets/answer_howdoi_all.png
+++ b/api/bots/howdoi/assets/answer_howdoi_all.png
--- a/api/bots/howdoi/assets/answer_howdowe.png
+++ b/api/bots/howdoi/assets/answer_howdowe.png
--- a/api/bots/howdoi/assets/question_howdoi_all.png
+++ b/api/bots/howdoi/assets/question_howdoi_all.png
--- a/api/bots/howdoi/assets/question_howdowe.png
+++ b/api/bots/howdoi/assets/question_howdowe.png
--- a/api/bots/howdoi/howdoi.py
+++ b/api/bots/howdoi/howdoi.py
@@ -1,118 +0,0 @@
-"""
-This bot uses the python library `howdoi` which is not a
-dependency of Zulip. To use this module, you will have to
-install it in your local machine. In your terminal, enter
-the following command:
-
-    $ sudo pip install howdoi --upgrade
-
-Note:
-    * You might have to use `pip3` if you are using python 3.
-    * The install command would also download any dependency
-      required by `howdoi`.
-"""
-
-from __future__ import absolute_import
-
-import sys
-import logging
-from textwrap import fill
-try:
-    from howdoi.howdoi import howdoi
-except ImportError:
-    logging.error("Dependency missing!!\n%s" % (__doc__))
-    sys.exit(0)
-
-
-class HowdoiHandler(object):
-    '''
-    This plugin facilitates searching Stack Overflow for
-    techanical answers based on the Python library `howdoi`.
-    To get the best possible answer, only include keywords
-    in your questions.
-
-    There are two possible commands:
-    * @mention-bot howdowe > This would return the answer to the same
-                 stream that it was called from.
-
-    * @mention-bot howdoi > The bot would send a private message to the
-                user containing the answer.
-
-    By default, howdoi only returns the coding section of the
-    first search result if possible, to see the full answer
-    from Stack Overflow, append a '!' to the commands.
-    (ie '@mention-bot howdoi!', '@mention-bot howdowe!')
-    '''
-
-    MAX_LINE_LENGTH = 85
-
-    def usage(self):
-        return '''
-            This plugin will allow users to get techanical
-            answers from Stackoverflow. Users should preface
-            their questions with one of the following:
-
-                * @mention-bot howdowe > Answer to the same stream
-                * @mention-bot howdoi > Answer via private message
-
-                * @mention-bot howdowe! OR @mention-bot howdoi! > Full answer from SO
-            '''
-
-    def line_wrap(self, string, length):
-        lines = string.split("\n")
-
-        wrapped = [(fill(line) if len(line) > length else line)
-                   for line in lines]
-
-        return "\n".join(wrapped).strip()
-
-    def get_answer(self, command, query):
-        question = query[len(command):].strip()
-        result = howdoi(dict(
-            query=question,
-            num_answers=1,
-            pos=1,
-            all=command[-1] == '!',
-            color=False
-        ))
-        _answer = self.line_wrap(result, HowdoiHandler.MAX_LINE_LENGTH)
-
-        answer = "Answer to '%s':\n```\n%s\n```" % (question, _answer)
-
-        return answer
-
-    def handle_message(self, message, client, state_handler):
-        question = message['content'].strip()
-
-        if question.startswith('howdowe!'):
-            client.send_message(dict(
-                type='stream',
-                to=message['display_recipient'],
-                subject=message['subject'],
-                content=self.get_answer('howdowe!', question)
-            ))
-
-        elif question.startswith('howdoi!'):
-            client.send_message(dict(
-                type='private',
-                to=message['sender_email'],
-                content=self.get_answer('howdoi!', question)
-            ))
-
-        elif question.startswith('howdowe'):
-            client.send_message(dict(
-                type='stream',
-                to=message['display_recipient'],
-                subject=message['subject'],
-                content=self.get_answer('howdowe', question)
-            ))
-
-        elif question.startswith('howdoi'):
-            client.send_message(dict(
-                type='private',
-                to=message['sender_email'],
-                content=self.get_answer('howdoi', question)
-            ))
-
-
-handler_class = HowdoiHandler
--- a/api/bots/howdoi/readme.md
+++ b/api/bots/howdoi/readme.md
@@ -1,56 +0,0 @@
-# Howdoi bot
-
-This bot will allow users to get technical answers from
-[StackOverflow](https://stackoverflow.com). It is build on top of the
-python command line tool [howdoi](https://github.com/gleitz/howdoi) by
-Benjamin Gleitzman.
-
-## Usage
-
-Simply prepend your questions with one of the following commands. The
-answer will be formatted differently depending the chosen command.
-
-| Command     | Respond                                                |
-| ----------- | ------------------------------------------------------ |
-| `@howdowe`  | Concise answer to the same stream.                     |
-| `@howdowe!` | Same as `@howdowe` but with full answer and URL of the solutions. |
-| `@howdoi`   | Concise answer replied to sender via private message.  |
-| `@howdoi!`  | Same as `@howdoi` but with full answer and URL of the solutions. |
-
-## Screenshots
-
-#### Example 1
-
-Question -> `@howdowe use supervisor in elixir`
-
-  ![howdowe question](assets/question_howdowe.png)
-
-Answer ->  Howdoi would try to **only** respond with the coding section
-of the answer.
-
-  ![howdowe answer](assets/answer_howdowe.png)
-
-#### Example 2
-
-Question -> `@howdoi! stack vs heap`
-
-  ![howdoi! question](assets/question_howdoi_all.png)
-
-Answer -> Howdoi would return the **full** stackoverflow answer via
-**private message** to the original sender. The URL of the answer can be
-seen at the bottom of the message.
-
-  ![howdoi! answer](assets/answer_howdoi_all.png)
-
-**Note:**
-
-* Line wrapped is enabled with a maximum line length of 85 characters.
-This could be adjusted in the source code (`HowdoiHandler.MAX_LINE_LENGTH`).
-
-* *Howdoi* generally perform better if you ask a question using keywords
-instead of a complete sentences (eg: "How do i make a decorator in Python"
-> "python decorator").
-
-* __[*Limitation*]__ If a answer contains multiple code blocks, the `@howdoi`
-and `@howdowe` commands would only return the first coding section, use
-`@howdo[we|i]!` in that case.
--- a/api/bots/incrementor/init.py
+++ b/api/bots/incrementor/init.py
--- a/api/bots/incrementor/incrementor.py
+++ b/api/bots/incrementor/incrementor.py
@@ -1,30 +0,0 @@
-# See readme.md for instructions on running this code.
-
-
-class IncrementorHandler(object):
-
-    def __init__(self):
-        self.number = 0
-        self.message_id = None
-
-    def usage(self):
-        return '''
-        This is a boilerplate bot that makes use of the
-        update_message function. For the first @-mention, it initially
-        replies with one message containing a `1`. Every time the bot
-        is @-mentioned, this number will be incremented in the same message.
-        '''
-
-    def handle_message(self, message, client, state_handler):
-        self.number += 1
-        if self.message_id is None:
-            result = client.send_reply(message, str(self.number))
-            self.message_id = result['id']
-        else:
-            client.update_message(dict(
-                message_id=self.message_id,
-                content=str(self.number),
-            ))
-
-
-handler_class = IncrementorHandler
--- a/api/bots/incrementor/readme.md
+++ b/api/bots/incrementor/readme.md
@@ -1,6 +0,0 @@
-# Incrementor bot
-
-This is a boilerplate bot that makes use of the
-update_message function. For the first @-mention, it initially
-replies with one message containing a `1`. Every time the bot
-is @-mentioned, this number will be incremented in the same message.
--- a/api/bots/john/init.py
+++ b/api/bots/john/init.py
--- a/Show More
+++ b/Show More