Release Zulip Server 1.5.2.

invite: Fix invite_by_admins_only to be enforced in backend.
Apparently, this setting never actually was wired up to anything other than hiding the UI widget. Huge thanks to Ibram Marzouk from the HackerOne community for finding this security bug.
2025-10-23 04:52:12 +00:00 · 2017-06-01 14:24:51 -07:00 · 2017-05-18 11:10:50 -07:00 · 2017-05-18 11:07:33 -07:00 · 2017-05-18 11:05:42 -07:00 · 2017-02-07 11:24:57 -08:00
4972 changed files with 200877 additions and 550745 deletions
--- a/.browserslistrc
+++ b/.browserslistrc
@@ -1,6 +0,0 @@
-> 0.2%
-> 0.2% in US
-last 2 versions
-Firefox ESR
-not dead
-Chrome 26  # similar to PhantomJS
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,151 +0,0 @@
-# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
-#   high-level documentation on our CircleCI setup.
-# See CircleCI upstream's docs on this config format:
-#   https://circleci.com/docs/2.0/language-python/
-#
-version: 2.0
-aliases:
-  - &create_cache_directories
-    run:
-      name: create cache directories
-      command: |
-          dirs=(/srv/zulip-{npm,venv}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R circleci "${dirs[@]}"
-
-  - &restore_cache_package_json
-    restore_cache:
-      keys:
-      - v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-
-  - &restore_cache_requirements
-    restore_cache:
-      keys:
-      - v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-  - &install_dependencies
-    run:
-      name: install dependencies
-      command: |
-        sudo apt-get update
-        # Install moreutils so we can use `ts` and `mispipe` in the following.
-        sudo apt-get install -y moreutils
-
-        # CircleCI sets the following in Git config at clone time:
-        #   url.ssh://git@github.com.insteadOf https://github.com
-        # This breaks the Git clones in the NVM `install.sh` we run
-        # in `install-node`.
-        # TODO: figure out why that breaks, and whether we want it.
-        #   (Is it an optimization?)
-        rm -f /home/circleci/.gitconfig
-
-        # This is the main setup job for the test suite
-        mispipe "tools/ci/setup-backend" ts
-
-        # Cleaning caches is mostly unnecessary in Circle, because
-        # most builds don't get to write to the cache.
-        # mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
-
-  - &save_cache_package_json
-    save_cache:
-      paths:
-        - /srv/zulip-npm-cache
-      key: v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-
-  - &save_cache_requirements
-    save_cache:
-      paths:
-        - /srv/zulip-venv-cache
-      key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-    # TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
-
-  - &run_backend_tests
-    run:
-      name: run backend tests
-      command: |
-        . /srv/zulip-py3-venv/bin/activate
-        mispipe ./tools/ci/backend ts
-
-  - &run_frontend_tests
-    run:
-      name: run frontend tests
-      command: |
-        . /srv/zulip-py3-venv/bin/activate
-        mispipe ./tools/ci/frontend ts
-
-  - &upload_coverage_report
-    run:
-     name: upload coverage report
-     command: |
-       . /srv/zulip-py3-venv/bin/activate
-       pip install codecov && codecov \
-         || echo "Error in uploading coverage reports to codecov.io."
-
-jobs:
-  "xenial-backend-frontend-python3.5":
-    docker:
-      # This is built from tools/circleci/images/xenial/Dockerfile .
-      # Xenial ships with Python 3.5.
-      - image: gregprice/circleci:xenial-python-4.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *create_cache_directories
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *install_dependencies
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *run_backend_tests
-      - *run_frontend_tests
-      # We only need to upload coverage reports on whichever platform
-      # runs the frontend tests.
-      - *upload_coverage_report
-
-      - store_artifacts:
-          path: ./var/casper/
-          destination: casper
-
-      - store_artifacts:    
-          path: ../../../tmp/zulip-test-event-log/
-          destination: test-reports
-
-      - store_test_results:
-            path: ./var/xunit-test-results/casper/
-
-  "bionic-backend-python3.6":
-    docker:
-      # This is built from tools/circleci/images/bionic/Dockerfile .
-      # Bionic ships with Python 3.6.
-      - image: gregprice/circleci:bionic-python-1.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *create_cache_directories
-
-      - run:
-          name: do Bionic hack
-          command: |
-              # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
-              # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
-              sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
-
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *install_dependencies
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *run_backend_tests
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - "xenial-backend-frontend-python3.5"
-      - "bionic-backend-python3.6"
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,12 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        # Codecov has the tendency to report a lot of false negatives,
-        # so we basically suppress comments completely.
-        threshold: 50%
-        base: auto
-    patch: off
--- a/.coveralls.yml
+++ b/.coveralls.yml
@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
--- a/.editorconfig
+++ b/.editorconfig
@@ -6,20 +6,14 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true

-[*.{sh,py,pyi,js,ts,json,yml,xml,css,md,markdown,handlebars,html}]
+[*.{sh,py,js, json,yml,xml, css, md,markdown, handlebars,html}]
 indent_style = space
 indent_size = 4

-[*.py]
-max_line_length = 110
-
-[*.{js,ts}]
-max_line_length = 100
-
 [*.{svg,rb,pp,pl}]
 indent_style = space
 indent_size = 2

-[*.cfg]
+[*.{cfg}]
 indent_style = space
 indent_size = 8
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,10 +1,3 @@
-# This is intended for generated files and vendored third-party files.
-# For our source code, instead of adding files here, consider using
-# specific eslint-disable comments in the files themselves.
-
-/docs/_build
-/static/generated
-/static/third
-/static/webpack-bundles
-/var
-/zulip-py3-venv
+static/js/bundle.js
+static/js/blueslip.js
+puppet/zulip_ops/files/statsd/local.js
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,496 +1,246 @@
 {
    "env": {
-        "node": true,
-        "es6": true
-    },
-    "parserOptions": {
-        "ecmaVersion": 2019,
-        "warnOnUnsupportedTypeScriptVersion": false,
-        "sourceType": "module"
+        "node": true
    },
    "globals": {
        "$": false,
-        "ClipboardJS": false,
-        "Dict": false,
-        "FetchStatus": false,
-        "Filter": false,
-        "Handlebars": false,
-        "LightboxCanvas": false,
-        "MessageListData": false,
-        "MessageListView": false,
-        "Plotly": false,
-        "SockJS": false,
-        "Socket": false,
-        "Sortable": false,
-        "WinChan": false,
-        "XDate": false,
        "_": false,
-        "activity": false,
-        "admin": false,
-        "alert_words": false,
-        "alert_words_ui": false,
-        "attachments_ui": false,
-        "avatar": false,
-        "billing": false,
-        "blueslip": false,
-        "bot_data": false,
-        "bridge": false,
-        "buddy_data": false,
-        "buddy_list": false,
-        "channel": false,
-        "click_handlers": false,
-        "color_data": false,
-        "colorspace": false,
-        "common": false,
-        "components": false,
-        "compose": false,
-        "compose_actions": false,
-        "compose_fade": false,
-        "compose_pm_pill": false,
-        "compose_state": false,
-        "compose_ui": false,
-        "composebox_typeahead": false,
-        "condense": false,
-        "confirm_dialog": false,
-        "copy_and_paste": false,
-        "csrf_token": false,
-        "current_msg_list": true,
-        "drafts": false,
-        "echo": false,
-        "emoji": false,
-        "emoji_codes": false,
-        "emoji_picker": false,
-        "favicon": false,
-        "feature_flags": false,
-        "feedback_widget": false,
-        "fenced_code": false,
-        "flatpickr": false,
-        "floating_recipient_bar": false,
-        "gear_menu": false,
-        "hash_util": false,
-        "hashchange": false,
-        "helpers": false,
-        "history": false,
-        "home_msg_list": false,
-        "hotspots": false,
-        "i18n": false,
-        "info_overlay": false,
-        "input_pill": false,
-        "invite": false,
        "jQuery": false,
-        "katex": false,
-        "keydown_util": false,
-        "lightbox": false,
-        "list_cursor": false,
-        "list_render": false,
-        "list_util": false,
-        "loading": false,
-        "localStorage": false,
-        "local_message": false,
-        "localstorage": false,
-        "location": false,
-        "markdown": false,
+        "Spinner": false,
+        "Handlebars": false,
+        "XDate": false,
+        "zxcvbn": false,
+        "LazyLoad": false,
+        "Dropbox": false,
+        "SockJS": false,
        "marked": false,
-        "md5": false,
-        "message_edit": false,
-        "message_events": false,
-        "message_fetch": false,
-        "message_flags": false,
-        "message_list": false,
-        "message_live_update": false,
-        "message_scroll": false,
-        "message_store": false,
-        "message_util": false,
-        "message_viewport": false,
-        "moment": false,
-        "muting": false,
-        "muting_ui": false,
-        "narrow": false,
-        "narrow_state": false,
-        "navigate": false,
-        "night_mode": false,
-        "notifications": false,
-        "overlays": false,
-        "padded_widget": false,
+        "i18n": false,
+        "bridge": false,
        "page_params": false,
-        "panels": false,
-        "people": false,
-        "pm_conversations": false,
-        "pm_list": false,
-        "pointer": false,
-        "popovers": false,
-        "presence": false,
-        "pygments_data": false,
-        "reactions": false,
-        "realm_icon": false,
-        "realm_logo": false,
-        "realm_night_logo": false,
-        "recent_senders": false,
-        "reload": false,
-        "reload_state": false,
-        "reminder": false,
-        "resize": false,
-        "rows": false,
-        "rtl": false,
-        "run_test": false,
-        "schema": false,
-        "scroll_bar": false,
-        "scroll_util": false,
-        "search": false,
-        "search_pill": false,
-        "search_pill_widget": false,
-        "search_suggestion": false,
-        "search_util": false,
-        "sent_messages": false,
-        "server_events": false,
-        "server_events_dispatch": false,
-        "settings": false,
-        "settings_account": false,
-        "settings_bots": false,
-        "settings_display": false,
-        "settings_emoji": false,
-        "settings_exports": false,
-        "settings_linkifiers": false,
-        "settings_invites": false,
-        "settings_muting": false,
-        "settings_notifications": false,
-        "settings_org": false,
-        "settings_panel_menu": false,
-        "settings_profile_fields": false,
-        "settings_sections": false,
-        "settings_streams": false,
-        "settings_toggle": false,
-        "settings_ui": false,
-        "settings_user_groups": false,
-        "settings_users": false,
-        "starred_messages": false,
-        "stream_color": false,
-        "stream_create": false,
-        "stream_data": false,
-        "stream_edit": false,
-        "stream_events": false,
-        "stream_list": false,
-        "stream_muting": false,
-        "stream_popover": false,
-        "stream_sort": false,
-        "stream_ui_updates": false,
-        "StripeCheckout": false,
-        "submessage": false,
-        "subs": false,
-        "tab_bar": false,
-        "templates": false,
-        "tictactoe_widget": false,
-        "timerender": false,
-        "todo_widget": false,
-        "top_left_corner": false,
-        "topic_data": false,
-        "topic_generator": false,
-        "topic_list": false,
-        "topic_zoom": false,
-        "transmit": false,
-        "tutorial": false,
+        "status_classes": false,
+        "password_quality": false,
+        "csrf_token": false,
        "typeahead_helper": false,
-        "typing": false,
-        "typing_data": false,
-        "typing_events": false,
+        "popovers": false,
+        "server_events": false,
        "ui": false,
-        "ui_init": false,
-        "ui_report": false,
-        "ui_util": false,
-        "unread": false,
-        "unread_ops": false,
-        "unread_ui": false,
-        "upgrade": false,
-        "upload": false,
-        "upload_widget": false,
-        "user_events": false,
-        "user_groups": false,
-        "user_pill": false,
-        "user_search": false,
-        "user_status": false,
-        "user_status_ui": false,
+        "stream_color": false,
+        "people": false,
+        "navigate": false,
+        "settings": false,
+        "resize": false,
+        "loading": false,
+        "compose": false,
+        "compose_fade": false,
+        "subs": false,
+        "timerender": false,
+        "message_live_update": false,
+        "message_edit": false,
+        "reload": false,
+        "composebox_typeahead": false,
+        "search": false,
+        "topic_list": false,
+        "gear_menu": false,
+        "hashchange": false,
+        "message_list": false,
+        "Filter": false,
+        "pointer": false,
        "util": false,
-        "poll_widget": false,
-        "widgetize": false,
-        "zcommand": false,
-        "zform": false,
-        "zxcvbn": false
+        "MessageListView": false,
+        "blueslip": false,
+        "rows": false,
+        "WinChan": false,
+        "muting_ui": false,
+        "Socket": false,
+        "channel": false,
+        "components": false,
+        "viewport": false,
+        "avatar": false,
+        "feature_flags": false,
+        "search_suggestion": false,
+        "referral": false,
+        "notifications": false,
+        "message_flags": false,
+        "bot_data": false,
+        "stream_list": false,
+        "narrow": false,
+        "admin": false,
+        "stream_data": false,
+        "muting": false,
+        "Dict": false,
+        "unread": false,
+        "alert_words_ui": false,
+        "message_store": false,
+        "favicon": false,
+        "condense": false,
+        "floating_recipient_bar": false,
+        "tab_bar": false,
+        "emoji": false,
+        "activity": false,
+        "invite": false,
+        "colorspace": false,
+        "reactions": false,
+        "tutorial": false,
+        "templates": false,
+        "alert_words": false,
+        "fenced_code": false,
+        "echo": false,
+        "localstorage": false,
+        "current_msg_list": true,
+        "home_msg_list": false,
+        "pm_list": false,
+        "unread_ui": false,
+        "user_events": false,
+        "Plotly": false,
+        "emoji_codes": false
    },
-    "plugins": [
-        "eslint-plugin-empty-returns"
-    ],
    "rules": {
-        "array-callback-return": "error",
-        "array-bracket-spacing": "error",
-        "arrow-spacing": [ "error", { "before": true, "after": true } ],
-        "block-scoped-var": "error",
-        "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
-        "camelcase": "off",
-        "comma-dangle": [ "error",
-            {
-                "arrays": "always-multiline",
-                "objects": "always-multiline",
-                "imports": "always-multiline",
-                "exports": "always-multiline",
-                "functions": "never"
-            }
-        ],
-        "comma-spacing": [ "error",
-            {
-                "before": false,
-                "after": true
-            }
-        ],
-        "complexity": [ "off", 4 ],
-        "curly": "error",
-        "dot-notation": [ "error", { "allowKeywords": true } ],
-        "empty-returns/main": "error",
-        "eol-last": [ "error", "always" ],
-        "eqeqeq": "error",
-        "func-style": [ "off", "expression" ],
-        "guard-for-in": "error",
-        "indent": ["error", 4, {
-            "ArrayExpression": "first",
-            "ObjectExpression": "first",
-            "SwitchCase": 0,
-            "CallExpression": {"arguments": "first"},
-            "FunctionExpression": {"parameters": "first"},
-            "FunctionDeclaration": {"parameters": "first"}
-        }],
-        "key-spacing": [ "error",
-            {
-                "beforeColon": false,
-                "afterColon": true
-            }
-        ],
-        "keyword-spacing": [ "error",
-            {
-                "before": true,
-                "after": true,
-                "overrides": {
-                    "return": { "after": true },
-                    "throw": { "after": true },
-                    "case": { "after": true }
-                }
-            }
-        ],
-        "max-depth": [ "off", 4 ],
-        "max-len": [ "error", 100, 2,
-            {
-                "ignoreUrls": true,
-                "ignoreComments": false,
-                "ignoreRegExpLiterals": true,
-                "ignoreStrings": true,
-                "ignoreTemplateLiterals": true
-            }
-        ],
-        "max-params": [ "off", 3 ],
-        "max-statements": [ "off", 10 ],
-        "new-cap": [ "error",
-            {
-                "newIsCap": true,
-                "capIsNew": false
-            }
-        ],
-        "new-parens": "error",
-        "newline-per-chained-call": "off",
-        "no-alert": "error",
-        "no-array-constructor": "error",
-        "no-bitwise": "error",
-        "no-caller": "error",
+        "no-restricted-syntax": 0,
+        "no-nested-ternary": 0,
+        "spaced-comment": 0,
+        "space-infix-ops": 0,
+        "newline-per-chained-call": 0,
+        "no-whitespace-before-property": 0,
+        "padded-blocks": 0,
+        "space-in-parens": 0,
+        "eol-last": ["error", "always"],
+        "no-unneeded-ternary": ["error", { "defaultAssignment": false }],
        "no-case-declarations": "error",
-        "no-catch-shadow": "error",
-        "no-console": "off",
-        "no-const-assign": "error",
-        "no-control-regex": "error",
-        "no-debugger": "error",
-        "no-delete-var": "error",
-        "no-div-regex": "error",
-        "no-dupe-class-members": "error",
-        "no-dupe-keys": "error",
+        "eqeqeq": ["error", "allow-null"],
        "no-duplicate-imports": "error",
-        "no-else-return": "error",
-        "no-empty": "error",
-        "no-empty-character-class": "error",
-        "no-eq-null": "error",
-        "no-eval": "error",
-        "no-ex-assign": "error",
-        "no-extra-parens": ["error", "all"],
-        "no-extra-semi": "error",
-        "no-fallthrough": "error",
-        "no-floating-decimal": "error",
-        "no-func-assign": "error",
-        "no-implied-eval": "error",
-        "no-iterator": "error",
-        "no-label-var": "error",
-        "no-labels": "error",
-        "no-loop-func": "error",
-        "no-mixed-requires": [ "off", false ],
-        "no-multi-str": "error",
-        "no-native-reassign": "error",
-        "no-nested-ternary": "off",
-        "no-new-func": "error",
-        "no-new-object": "error",
-        "no-new-wrappers": "error",
-        "no-obj-calls": "error",
-        "no-octal": "error",
-        "no-octal-escape": "error",
-        "no-param-reassign": "off",
-        "no-plusplus": "error",
-        "no-proto": "error",
-        "no-redeclare": "error",
-        "no-regex-spaces": "error",
-        "no-restricted-syntax": "off",
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-self-compare": "error",
-        "no-shadow": "off",
-        "no-sync": "error",
-        "no-ternary": "off",
-        "no-trailing-spaces": "error",
        "no-undef": "error",
-        "no-undef-init": "error",
-        "no-underscore-dangle": "off",
-        "no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
-        "no-unreachable": "error",
-        "no-unused-expressions": "error",
-        "no-unused-vars": [ "error",
-            {
-                "vars": "local",
-                "args": "after-used",
-                "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
-            }
-        ],
-        "no-use-before-define": "error",
+        "dot-notation": ["error", { "allowKeywords": true }],
+        "no-iterator": "error",
+        "no-dupe-class-members": "error",
        "no-useless-constructor": "error",
-        // The Zulip codebase complies partially with the "no-useless-escape"
-        // rule; only regex expressions haven't been updated yet.
-        // Updated regex expressions are currently being tested in casper
-        // files and will decide about a potential future enforcement of this rule.
-        "no-useless-escape": "off",
-        "no-var": "error",
-        "space-unary-ops": "error",
-        "no-whitespace-before-property": "error",
-        "no-with": "error",
-        "one-var": [ "error", "never" ],
-        "padded-blocks": "off",
-        "prefer-const": [ "error",
-            {
-                "destructuring": "any",
-                "ignoreReadBeforeAssign": true
-            }
-        ],
-        "quote-props": [ "error", "as-needed",
-            {
-                "keywords": false,
-                "unnecessary": true,
-                "numbers": false
-            }
-        ],
-        "quotes": [ "off", "single" ],
-        "radix": "error",
-        "semi": "error",
-        "semi-spacing": ["error", {"before": false, "after": true}],
-        "sort-imports": "error",
-        "space-before-blocks": "error",
-        "space-before-function-paren": [ "error",
-            {
-                "anonymous": "always",
-                "named": "never",
-                "asyncArrow": "always"
-            }
-        ],
-        "space-in-parens": "error",
-        "space-infix-ops": "error",
-        "spaced-comment": "off",
-        "strict": "off",
+        "prefer-const": ["error", {
+            "destructuring": "any",
+            "ignoreReadBeforeAssign": true
+        }],
+        "no-const-assign": "error",
+        "no-new-object": 2,
+        "quote-props": ["error", "as-needed", {
+            "keywords": false,
+            "unnecessary": true,
+            "numbers": false
+        }],
+        "no-array-constructor": "error",
+        "array-callback-return": "error",
        "template-curly-spacing": "error",
-        "unnecessary-strict": "off",
-        "use-isnan": "error",
-        "valid-typeof": [ "error", { "requireStringLiterals": true } ],
-        "wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
-        "wrap-regex": "off",
-        "yoda": "error"
-    },
-    "overrides": [
-        {
-            "files": [
-                "frontend_tests/casper_tests/*.js",
-                "frontend_tests/casper_lib/*.js"
-            ],
-            "rules": {
-                "no-var": "off" // PhantomJS doesn’t support let, const
+        //The Zulip codebase complies partially with the "no-useless-escape" rule; only regex expressions haven't been updated yet.
+        //Updated regex expressions are currently being tested in casper files and will decide about a potential future enforcement of this rule.
+        "no-useless-escape": 0,
+        "func-style": ["off", "expression"],
+        "wrap-iife": ["error", "outside", { "functionPrototypeMethods": false }],
+        "no-new-func": "error",
+        "space-before-function-paren": ["error", { "anonymous": "always", "named": "never", "asyncArrow": "always" }],
+        "no-param-reassign": 0,
+        "prefer-spread": "error",
+        "arrow-spacing": ["error", { "before": true, "after": true }],
+        "no-alert": 2,
+        "no-array-constructor": 2,
+        "no-caller": 2,
+        "no-bitwise": 2,
+        "no-catch-shadow": 2,
+        "comma-dangle": ["error", {
+            "arrays": "always-multiline",
+            "objects": "always-multiline",
+            "imports": "always-multiline",
+            "exports": "always-multiline",
+            "functions": "never"
+        }],
+        "no-console": 0,
+        "no-control-regex": 2,
+        "no-debugger": 2,
+        "no-div-regex": 2,
+        "no-dupe-keys": 2,
+        "no-else-return": 2,
+        "no-empty": 2,
+        "no-empty-character-class": 2,
+        "no-eq-null": 2,
+        "no-eval": 2,
+        "no-ex-assign": 2,
+        "no-extra-semi": 2,
+        "no-func-assign": 2,
+        "no-floating-decimal": 2,
+        "no-implied-eval": 2,
+        "no-with": 2,
+        "no-fallthrough": 2,
+        "no-unreachable": 2,
+        "no-undef": 2,
+        "no-undef-init": 2,
+        "no-unused-expressions": 2,
+        "no-octal": 2,
+        "no-octal-escape": 2,
+        "no-obj-calls": 2,
+        "no-multi-str": 2,
+        "no-new-wrappers": 2,
+        "no-new": 2,
+        "no-new-func": 2,
+        "no-native-reassign": 2,
+        "no-plusplus": 2,
+        "no-delete-var": 2,
+        "no-return-assign": 2,
+        "no-new-object": 2,
+        "no-label-var": 2,
+        "no-ternary": 0,
+        "no-self-compare": 2,
+        "no-sync": 2,
+        "no-underscore-dangle": 0,
+        "no-loop-func": 2,
+        "no-labels": 2,
+        "no-unused-vars": ["error", { "vars": "local", "args": "after-used",
+                                      "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
+                                    }],
+        "no-script-url": 2,
+        "no-proto": 2,
+        "no-iterator": 2,
+        "no-mixed-requires": [0, false],
+        "no-extra-parens": ["error", "functions"],
+        "no-shadow": 0,
+        "no-use-before-define": 2,
+        "no-redeclare": 2,
+        "no-regex-spaces": 2,
+        "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
+        "block-scoped-var": 2,
+        "camelcase": 0,
+        "complexity": [0, 4],
+        "curly": 2,
+        "dot-notation": 2,
+        "guard-for-in": 2,
+        "max-depth": [0, 4],
+        "max-len": ["error", 100, 2, {
+            "ignoreUrls": true,
+            "ignoreComments": false,
+            "ignoreRegExpLiterals": true,
+            "ignoreStrings": true,
+            "ignoreTemplateLiterals": true
+        }],
+        "max-params": [0, 3],
+        "max-statements": [0, 10],
+        "new-cap": ["error", { "newIsCap": true, "capIsNew": false }],
+        "new-parens": 2,
+        "one-var": ["error", "never"],
+        "quotes": [0, "single"],
+        "quote-props": ["error", "as-needed", { "keywords": false, "unnecessary": true, "numbers": false }],
+        "radix": 2,
+        "semi": 2,
+        "keyword-spacing": ["error", {
+            "before": true,
+            "after": true,
+            "overrides": {
+                "return": { "after": true },
+                "throw": { "after": true },
+                "case": { "after": true }
            }
-        },
-        {
-            "files": ["**/*.ts"],
-            "parser": "@typescript-eslint/parser",
-            "parserOptions": {
-                "project": "tsconfig.json"
-            },
-            "plugins": ["@typescript-eslint"],
-            "rules": {
-                // Disable base rule to avoid conflict
-                "empty-returns/main": "off",
-                "indent": "off",
-                "func-call-spacing": "off",
-                "no-magic-numbers": "off",
-                "semi": "off",
-                "no-unused-vars": "off",
-                "no-useless-constructor": "off",
-
-                "@typescript-eslint/adjacent-overload-signatures": "error",
-                "@typescript-eslint/array-type": "error",
-                "@typescript-eslint/await-thenable": "error",
-                "@typescript-eslint/ban-types": "error",
-                "@typescript-eslint/ban-ts-ignore": "off",
-                "@typescript-eslint/camelcase": "off",
-                "@typescript-eslint/class-name-casing": "error",
-                "@typescript-eslint/consistent-type-assertions": "error",
-                "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
-                "@typescript-eslint/explicit-member-accessibility": "off",
-                "@typescript-eslint/func-call-spacing": "error",
-                "@typescript-eslint/generic-type-naming": "off",
-                "@typescript-eslint/indent": "error",
-                "@typescript-eslint/interface-name-prefix": "off",
-                "@typescript-eslint/member-delimiter-style": "error",
-                "@typescript-eslint/member-naming": ["error", { "private": "^_" } ],
-                "@typescript-eslint/member-ordering": "error",
-                "@typescript-eslint/no-array-constructor": "error",
-                "@typescript-eslint/no-empty-interface": "error",
-                "@typescript-eslint/no-explicit-any": "off",
-                "@typescript-eslint/no-extraneous-class": "error",
-                "@typescript-eslint/no-for-in-array": "off",
-                "@typescript-eslint/no-inferrable-types": "error",
-                "@typescript-eslint/no-magic-numbers": "off",
-                "@typescript-eslint/no-misused-new": "error",
-                "@typescript-eslint/no-namespace": "error",
-                "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-parameter-properties": "error",
-                "@typescript-eslint/no-require-imports": "off",
-                "@typescript-eslint/no-this-alias": "off",
-                "@typescript-eslint/no-type-alias": "off",
-                "@typescript-eslint/no-unnecessary-qualifier": "error",
-                "@typescript-eslint/no-unnecessary-type-assertion": "error",
-                "@typescript-eslint/no-unused-vars": ["error", { "varsIgnorePattern": "^_" } ],
-                "@typescript-eslint/no-use-before-define": "error",
-                "@typescript-eslint/no-useless-constructor": "error",
-                "@typescript-eslint/no-var-requires": "off",
-                "@typescript-eslint/prefer-for-of": "off",
-                "@typescript-eslint/prefer-function-type": "off",
-                "@typescript-eslint/prefer-includes": "error",
-                "@typescript-eslint/prefer-interface": "off",
-                "@typescript-eslint/prefer-namespace-keyword": "error",
-                "@typescript-eslint/prefer-regexp-exec": "error",
-                "@typescript-eslint/prefer-string-starts-ends-with": "error",
-                "@typescript-eslint/promise-function-async": "error",
-                "@typescript-eslint/restrict-plus-operands": "off",
-                "@typescript-eslint/semi": "error",
-                "@typescript-eslint/triple-slash-reference": "error",
-                "@typescript-eslint/type-annotation-spacing": "error",
-                "@typescript-eslint/unbound-method": "off",
-                "@typescript-eslint/unified-signatures": "error"
-            }
-        }
-    ]
+        }],
+        "space-before-blocks": 2,
+        "strict": 0,
+        "unnecessary-strict": 0,
+        "use-isnan": 2,
+        "valid-typeof": ["error", { "requireStringLiterals": true }],
+        "wrap-iife": 2,
+        "wrap-regex": 0,
+        "yoda": 2
+    }
 }
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,13 +1,29 @@
 *   text=auto eol=lf
 *.gif binary
 *.jpg binary
-*.jpeg binary
 *.eot binary
 *.woff binary
-*.woff2 binary
 *.svg binary
 *.ttf binary
 *.png binary
 *.otf binary
 *.tif binary
-*.ogg binary
+.gitignore export-ignore
+.gitattributes export-ignore
+/static/assets export-ignore
+/bots export-ignore
+/corporate export-ignore
+/static export-ignore
+/tools export-ignore
+/zilencer export-ignore
+/templates/corporate export-ignore
+/templates/zilencer export-ignore
+/puppet/zulip_internal export-ignore
+/zproject/local_settings.py export-ignore
+/zproject/test_settings.py export-ignore
+/zerver/fixtures export-ignore
+/zerver/tests export-ignore
+/frontend_tests export-ignore
+/node_modules export-ignore
+/humbug export-ignore
+/locale export-ignore
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,14 +0,0 @@
-<!-- What's this PR for?  (Just a link to an issue is fine.) -->
-
-
-**Testing Plan:** <!-- How have you tested? -->
-
-
-**GIFs or Screenshots:** <!-- If a UI change.  See:
-  https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
-  -->
-
-
-<!-- Also be sure to make clear, coherent commits:
-  https://zulip.readthedocs.io/en/latest/contributing/version-control.html
-  -->
--- a/.gitignore
+++ b/.gitignore
@@ -1,82 +1,29 @@
-# Quick format and style primer:
-#
-#  * If a pattern is meant only for a specific location, it should have a
-#    leading slash, like `/staticfiles.json`.
-#    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
-#      will do, but this makes a confusing pattern.  Adding a leading slash
-#      is clearer.
-#
-#  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
-#    match in any subdirectory.
-#
-#  * Subdirectories with several internal things to ignore get their own
-#    `.gitignore` files.
-#
-#  * Comments must be on their own line.  (Otherwise they don't work.)
-#
-# See `git help ignore` for details on the format.
-
-## Config files for the dev environment
-/zproject/dev-secrets.conf
-/tools/conf.ini
-/tools/custom_provision
-/tools/droplets/conf.ini
-
-## Byproducts of setting up and using the dev environment
 *.pyc
-package-lock.json
-
-/.vagrant
-/var
-
-/.dmypy.json
-
-# Dockerfiles generated for CircleCI
-/tools/circleci/images
-
-# Generated i18n data
-/locale/en
-/locale/language_options.json
-/locale/language_name_map.json
-/locale/*/mobile.json
-
-# Static build
-*.mo
-npm-debug.log
-/node_modules
-/prod-static
-/staticfiles.json
-/webpack-stats-production.json
-/yarn-error.log
-zulip-git-version
-
-# Test / analysis tools
-.coverage
-
-## Files (or really symlinks) created in a prod deployment
-/zproject/prod_settings.py
-/zulip-current-venv
-/zulip-py3-venv
-
-## Files left by various editors and local environments
-# (Ideally these should be in everyone's respective personal gitignore files.)
 *~
+/prod-static
 *.sw[po]
-.idea
+*.DS_Store
 .kdev4
+.idea
 zulip.kdev4
+coverage/
+.coverage
+/queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
+.vagrant
+/zproject/dev-secrets.conf
+static/js/bundle.js
+static/generated/emoji
+static/generated/github-contributors.json
+static/locale/language_options.json
+/node_modules
+/staticfiles.json
+npm-debug.log
+*.mo
+var/*
 .vscode/
-*.DS_Store
-# .cache/ is generated by VSCode's test runner
-.cache/
-.eslintcache
-
-## Miscellaneous
-# (Ideally this section is empty.)
-zthumbor/thumbor_local_settings.py
-.transifexrc
+tools/conf.ini
--- a/.gitlint
+++ b/.gitlint
@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing, title-imperative-mood
-
-extra-path=tools/lib/gitlint-rules.py
-
-[title-match-regex-allow-exception]
-regex=^(.+:\ )?[A-Z].+\.$
-
-[title-max-length]
-line-length=76
-
-[body-max-line-length]
-line-length=76
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,10 +0,0 @@
-[settings]
-line_length = 79
-multi_line_output = 2
-balanced_wrapping = true
-known_third_party = django, ujson, sqlalchemy
-known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
-sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
-lines_after_imports = 1
-# See the comment related to ioloop_logging for why this is skipped.
-skip = zerver/management/commands/runtornado.py
--- a/.stylelintrc
+++ b/.stylelintrc
@@ -1,67 +0,0 @@
-{
-    "rules": {
-        # Stylistic rules for CSS.
-        "function-comma-space-after": "always",
-        "function-comma-space-before": "never",
-        "function-max-empty-lines": 0,
-        "function-whitespace-after": "always",
-
-        "value-keyword-case": "lower",
-        "value-list-comma-newline-after": "always-multi-line",
-        "value-list-comma-space-after": "always-single-line",
-        "value-list-comma-space-before": "never",
-        "value-list-max-empty-lines": 0,
-
-        "unit-case": "lower",
-        "property-case": "lower",
-        "color-hex-case": "lower",
-
-        "declaration-bang-space-before": "always",
-        "declaration-colon-newline-after": "always-multi-line",
-        "declaration-colon-space-after": "always-single-line",
-        "declaration-colon-space-before": "never",
-        "declaration-block-semicolon-newline-after": "always",
-        "declaration-block-semicolon-space-before": "never",
-        "declaration-block-trailing-semicolon": "always",
-
-        "block-closing-brace-empty-line-before": "never",
-        "block-closing-brace-newline-after": "always",
-        "block-closing-brace-newline-before": "always",
-        "block-opening-brace-newline-after": "always",
-        "block-opening-brace-space-before": "always",
-
-        "selector-attribute-brackets-space-inside": "never",
-        "selector-attribute-operator-space-after": "never",
-        "selector-attribute-operator-space-before": "never",
-        "selector-combinator-space-after": "always",
-        "selector-combinator-space-before": "always",
-        "selector-descendant-combinator-no-non-space": true,
-        "selector-pseudo-class-parentheses-space-inside": "never",
-        "selector-pseudo-element-case": "lower",
-        "selector-pseudo-element-colon-notation": "double",
-        "selector-type-case": "lower",
-        "selector-list-comma-newline-after": "always",
-        "selector-list-comma-space-before": "never",
-
-        "media-feature-colon-space-after": "always",
-        "media-feature-colon-space-before": "never",
-        "media-feature-name-case": "lower",
-        "media-feature-parentheses-space-inside": "never",
-        "media-feature-range-operator-space-after": "always",
-        "media-feature-range-operator-space-before": "always",
-        "media-query-list-comma-newline-after": "always",
-        "media-query-list-comma-space-before": "never",
-
-        "at-rule-name-case": "lower",
-        "at-rule-name-space-after": "always",
-        "at-rule-semicolon-newline-after": "always",
-        "at-rule-semicolon-space-before": "never",
-
-        "comment-whitespace-inside": "always",
-        "indentation": 4,
-        
-        # Limit language features
-        "color-no-hex": true,
-        "color-named": "never",
-    }
-}
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,66 +1,54 @@
-# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
-# high-level documentation on our Travis CI setup.
-dist: xenial
+dist: trusty
+before_install:
+   - nvm install 0.10
 install:
-  # Disable sometimes-broken sources.list in Travis base images
-  - sudo rm -vf /etc/apt/sources.list.d/*
-  - sudo apt-get update
-
  # Disable Travis CI's built-in NVM installation
-  - mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
-
-  # Install codecov, the library for the code coverage reporting tool we use
-  # With a retry to minimize impact of transient networking errors.
-  - mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
-
-  # This is the main setup job for the test suite
-  - mispipe "tools/ci/setup-$TEST_SUITE" ts
-
-  # Clean any caches that are not in use to avoid our cache
-  # becoming huge.
-  - mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
-
-script:
-  # We unset GEM_PATH here as a hack to work around Travis CI having
-  # broken running their system puppet with Ruby.  See
-  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
-  - unset GEM_PATH
-  - mispipe "./tools/ci/$TEST_SUITE" ts
+  - mv ~/.nvm ~/.travis-nvm-disabled
+  - pip install coveralls
+  - tools/travis/setup-$TEST_SUITE
+  - tools/clean-venv-cache --travis
 cache:
-  yarn: true
-  apt: false
-  directories:
+  - apt: false
+  - directories:
    - $HOME/zulip-venv-cache
-    - $HOME/zulip-npm-cache
-    - $HOME/zulip-emoji-cache
+    - node_modules
    - $HOME/node
-    - $HOME/misc
 env:
  global:
-    - BOTO_CONFIG=/nonexistent
+    - COVERAGE_FILE=var/.coverage
+    - COVERALLS_PARALLEL=true
+    - COVERALLS_SERVICE_NAME=travis-pro
+    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
+    - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
 language: python
-# Our test suites generally run on Python 3.5, the version in
-# Ubuntu 16.04 xenial, which is the oldest OS release we support.
+python:
+  - "2.7"
+  - "3.4"
 matrix:
  include:
-    # Travis will actually run the jobs in the order they're listed here;
-    # that doesn't seem to be documented, but it's what we see empirically.
-    # We only get 4 jobs running at a time, so we try to make the first few
-    # the most likely to break.
-    - python: "3.5"
+    - python: "3.4"
+      env: TEST_SUITE=static-analysis
+    - python: "3.4"
      env: TEST_SUITE=production
-    # Other suites moved to CircleCI -- see .circleci/.
+    - python: "2.7"
+      env: TEST_SUITE=production
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
 sudo: required
+services:
+- docker
 addons:
  artifacts:
    paths:
-      # Casper debugging data (screenshots, etc.) is super useful for
-      # debugging test flakes.
      - $(ls var/casper/* | tr "\n" ":")
      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
-  postgresql: "9.5"
-  apt:
-    packages:
-      - moreutils
+  postgresql: "9.3"
 after_success:
-  - codecov
+  coveralls
+notifications:
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
--- a/.tx/config
+++ b/.tx/config
@@ -1,33 +1,14 @@
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant

 [zulip.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
+source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
+file_filter = static/locale/<lang>/LC_MESSAGES/django.po

 [zulip.translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[zulip.mobile]
-file_filter = locale/<lang>/mobile.json
-source_file = locale/en/mobile.json
-source_lang = en
-type = KEYVALUEJSON
-
-[zulip-test.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
-source_lang = en
-type = PO
-
-[zulip-test.translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
+source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
+file_filter = static/locale/<lang>/translations.json
--- a/.yarnrc
+++ b/.yarnrc
@@ -1 +0,0 @@
-ignore-scripts true
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,105 +0,0 @@
-# Zulip Code of Conduct
-
-Like the technical community as a whole, the Zulip team and community is
-made up of a mixture of professionals and volunteers from all over the
-world, working on every aspect of the mission, including mentorship,
-teaching, and connecting people.
-
-Diversity is one of our huge strengths, but it can also lead to
-communication issues and unhappiness. To that end, we have a few ground
-rules that we ask people to adhere to. This code applies equally to
-founders, mentors, and those seeking help and guidance.
-
-This isn't an exhaustive list of things that you can't do. Rather, take it
-in the spirit in which it's intended --- a guide to make it easier to enrich
-all of us and the technical communities in which we participate.
-
-## Expected Behavior
-
-The following behaviors are expected and requested of all community members:
-
-* Participate. In doing so, you contribute to the health and longevity of
-  the community.
-* Exercise consideration and respect in your speech and actions.
-* Attempt collaboration before conflict. Assume good faith.
-* Refrain from demeaning, discriminatory, or harassing behavior and speech.
-* Take action or alert community leaders if you notice a dangerous
-  situation, someone in distress, or violations of this code, even if they
-  seem inconsequential.
-* Community event venues may be shared with members of the public; be
-  respectful to all patrons of these locations.
-
-## Unacceptable Behavior
-
-The following behaviors are considered harassment and are unacceptable
-within the Zulip community:
-
-* Jokes or derogatory language that singles out members of any race,
-  ethnicity, culture, national origin, color, immigration status, social and
-  economic class, educational level, language proficiency, sex, sexual
-  orientation, gender identity and expression, age, size, family status,
-  political belief, religion, and mental and physical ability.
-* Violence, threats of violence, or violent language directed against
-  another person.
-* Disseminating or threatening to disseminate another person's personal
-  information.
-* Personal insults of any sort.
-* Posting or displaying sexually explicit or violent material.
-* Inappropriate photography or recording.
-* Deliberate intimidation, stalking, or following (online or in person).
-* Unwelcome sexual attention. This includes sexualized comments or jokes,
-  inappropriate touching or groping, and unwelcomed sexual advances.
-* Sustained disruption of community events, including talks and
-  presentations.
-* Advocating for, or encouraging, any of the behaviors above.
-
-## Reporting and Enforcement
-
-Harassment and other code of conduct violations reduce the value of the
-community for everyone. If someone makes you or anyone else feel unsafe or
-unwelcome, please report it to the community organizers at
-zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
-report either personally or anonymously.
-
-If a community member engages in unacceptable behavior, the community
-organizers may take any action they deem appropriate, up to and including a
-temporary ban or permanent expulsion from the community without warning (and
-without refund in the case of a paid event).
-
-If someone outside the development community (e.g. a user of the Zulip
-software) engages in unacceptable behavior that affects someone in the
-community, we still want to know. Even if we don't have direct control over
-the violator, the community organizers can still support the people
-affected, reduce the chance of a similar violation in the future, and take
-any direct action we can.
-
-The nature of reporting means it can only help after the fact. If you see
-something you can do while a violation is happening, do it. A lot of the
-harms of harassment and other violations can be mitigated by the victim
-knowing that the other people present are on their side.
-
-All reports will be kept confidential. In some cases, we may determine that a
-public statement will need to be made. In such cases, the identities of all
-victims and reporters will remain confidential unless those individuals
-instruct us otherwise.
-
-## Scope
-
-We expect all community participants (contributors, paid or otherwise,
-sponsors, and other guests) to abide by this Code of Conduct in all
-community venues, online and in-person, as well as in all private
-communications pertaining to community business.
-
-This Code of Conduct and its related procedures also applies to unacceptable
-behavior occurring outside the scope of community activities when such
-behavior has the potential to adversely affect the safety and well-being of
-community members.
-
-## License and Attribution
-
-This Code of Conduct is adapted from the
-[Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
-[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
-under a
-[Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
-license.
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,351 +0,0 @@
-# Contributing to Zulip
-
-Welcome to the Zulip community!
-
-## Community
-
-The
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-is the primary communication forum for the Zulip community. It is a good
-place to start whether you have a question, are a new contributor, are a new
-user, or anything else. Make sure to read the
-[community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
-before posting. The Zulip community is also governed by a
-[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
-
-You can subscribe to zulip-devel-announce@googlegroups.com or our
-[Twitter](https://twitter.com/zulip) account for a lower traffic (~1
-email/month) way to hear about things like mentorship opportunities with Google
-Code-in, in-person sprints at conferences, and other opportunities to
-contribute.
-
-## Ways to contribute
-
-To make a code or documentation contribution, read our
-[step-by-step guide](#your-first-codebase-contribution) to getting
-started with the Zulip codebase. A small sample of the type of work that
-needs doing:
-* Bug squashing and feature development on our Python/Django
-  [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip), React Native
-  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
-  [desktop app](https://github.com/zulip/zulip-desktop).
-* Building out our
-  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
-* [Writing an integration](https://zulipchat.com/api/integrations-overview).
-* Improving our [user](https://zulipchat.com/help/) or
-  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
-* [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
-  and manually testing pull requests.
-
-**Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. We list a few of them below:
-
-* [Reporting issues](#reporting-issues), including both feature requests and
-  bug reports.
-* [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
-* [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip.
-* [Outreach](#zulip-outreach): Star us on GitHub, upvote us
-  on product comparison sites, or write for [the Zulip blog](http://blog.zulip.org/).
-
-## Your first (codebase) contribution
-
-This section has a step by step guide to starting as a Zulip codebase
-contributor. It's long, but don't worry about doing all the steps perfectly;
-no one gets it right the first time, and there are a lot of people available
-to help.
-* First, make an account on the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
-  paying special attention to the community norms. If you'd like, introduce
-  yourself in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
-  your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken as you started using the
-  product.
-* Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-* [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-  getting help in
-  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
-  if you run into any troubles.
-* Read the
-  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
-  and do the Git tutorial (coming soon) if you are unfamiliar with
-  Git, getting help in
-  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
-  you run into any troubles.  Be sure to check out the
-  [extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
-* Sign the
-  [Dropbox Contributor License Agreement](https://opensource.dropbox.com/cla/).
-
-### Picking an issue
-
-Now, you're ready to pick your first issue! There are hundreds of open issues
-in the main codebase alone. This section will help you find an issue to work
-on.
-
-* If you're interested in
-  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
-  [desktop](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue),
-  or
-  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
-  development, check the respective links for open issues, or post in
-  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
-  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
-  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
-* For the main server and web repository, we recommend browsing
-  recently opened issues to look for issues you are confident you can
-  fix correctly in a way that clearly communicates why your changes
-  are the correct fix.  Our GitHub workflow bot, zulipbot, limits
-  users who have 0 commits merged to claiming a single issue labeled
-  with "good first issue" or "help wanted".
-* We also partition all of our issues in the main repo into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-* If the lists of issues are overwhelming, post in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
-  bit about your background and interests, and we'll help you out. The most
-  important thing to say is whether you're looking for a backend (Python),
-  frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
-  documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
-  bit about your programming experience and available time.
-
-We also welcome suggestions of features that you feel would be valuable or
-changes that you feel would make Zulip a better open source project. If you
-have a new feature you'd like to add, we recommend you start by posting in
-[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
-feature idea and the problem that you're hoping to solve.
-
-Other notes:
-* For a first pull request, it's better to aim for a smaller contribution
-  than a bigger one. Many first contributions have fewer than 10 lines of
-  changes (not counting changes to tests).
-* The full list of issues explicitly looking for a contributor can be
-  found with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  and
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels.  Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are confident you can resolve the
-  issue correctly and completely.  Issues without one of these labels
-  are fair game if Tim has written a clear technical design proposal
-  in the issue, or it is a bug that you can reproduce and you are
-  confident you can fix the issue correctly.
-* For most new contributors, there's a lot to learn while making your first
-  pull request. It's OK if it takes you a while; that's normal! You'll be
-  able to work a lot faster as you build experience.
-
-### Working on an issue
-
-To work on an issue, claim it by adding a comment with `@zulipbot claim` to
-the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
-workflow bot; it will assign you to the issue and label the issue as "in
-progress". Some additional notes:
-
-* You can only claim issues with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  or
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels. Zulipbot will give you an error if you try to claim an issue
-  without one of those labels.
-* You're encouraged to ask questions on how to best implement or debug your
-  changes -- the Zulip maintainers are excited to answer questions to help
-  you stay unblocked and working efficiently. You can ask questions on
-  chat.zulip.org, or on the GitHub issue or pull request.
-* We encourage early pull requests for work in progress. Prefix the title of
-  work in progress pull requests with `[WIP]`, and remove the prefix when
-  you think it might be mergeable and want it to be reviewed.
-* After updating a PR, add a comment to the GitHub thread mentioning that it
-  is ready for another review. GitHub only notifies maintainers of the
-  changes when you post a comment, so if you don't, your PR will likely be
-  neglected by accident!
-
-### And beyond
-
-A great place to look for a second issue is to look for issues with the same
-`area:` label as the last issue you resolved. You'll be able to reuse the
-work you did learning how that part of the codebase works. Also, the path to
-becoming a core developer often involves taking ownership of one of these area
-labels.
-
-## What makes a great Zulip contributor?
-
-Zulip runs a lot of [internship programs](#internship-programs), so we have
-a lot of experience with new contributors. In our experience, these are the
-best predictors of success:
-
-* Posting good questions. This generally means explaining your current
-  understanding, saying what you've done or tried so far, and including
-  tracebacks or other error messages if appropriate.
-* Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
-* Submitting carefully tested code. This generally means checking your work
-  through a combination of automated tests and manually clicking around the
-  UI trying to find bugs in your work. See
-  [things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
-  for additional ideas.
-* Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-* Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-* Being helpful and friendly on chat.zulip.org.
-
-These are also the main criteria we use to select interns for all of our
-internship programs.
-
-## Reporting issues
-
-If you find an easily reproducible bug and/or are experienced in reporting
-bugs, feel free to just open an issue on the relevant project on GitHub.
-
-If you have a feature request or are not yet sure what the underlying bug
-is, the best place to post issues is
-[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
-[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
-[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
-This allows us to interactively figure out what is going on, let you know if
-a similar issue has already been opened, and collect any other information
-we need. Choose a 2-4 word topic that describes the issue, explain the issue
-and how to reproduce it if known, your browser/OS if relevant, and a
-[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-if appropriate.
-
-**Reporting security issues**. Please do not report security issues
-  publicly, including on public streams on chat.zulip.org. You can email
-  zulip-security@googlegroups.com. We create a CVE for every security issue.
-
-## User feedback
-
-Nearly every feature we develop starts with a user request. If you are part
-of a group that is either using or considering using Zulip, we would love to
-hear about your experience with the product. If you're not sure what to
-write, here are some questions we're always very curious to know the answer
-to:
-
-* Evaluation: What is the process by which your organization chose or will
-  choose a group chat product?
-* Pros and cons: What are the pros and cons of Zulip for your organization,
-  and the pros and cons of other products you are evaluating?
-* Features: What are the features that are most important for your
-  organization? In the best case scenario, what would your chat solution do
-  for you?
-* Onboarding: If you remember it, what was your impression during your first
-  few minutes of using Zulip? What did you notice, and how did you feel? Was
-  there anything that stood out to you as confusing, or broken, or great?
-* Organization: What does your organization do? How big is the organization?
-  A link to your organization's website?
-
-## Internship programs
-
-Zulip runs internship programs with
-[Outreachy](https://www.outreachy.org/),
-[Google Summer of Code (GSoC)](https://developers.google.com/open-source/gsoc/)
-[1], and the
-[MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram),
-and has in the past taken summer interns from Harvard, MIT, and
-Stanford.
-
-While each third-party program has its own rules and requirements, the
-Zulip community's approaches all of these programs with these ideas in
-mind:
-* We try to make the application process as valuable for the applicant as
-  possible. Expect high quality code reviews, a supportive community, and
-  publicly viewable patches you can link to from your resume, regardless of
-  whether you are selected.
-* To apply, you'll have to submit at least one pull request to a Zulip
-  repository.  Most students accepted to one of our programs have
-  several merged pull requests (including at least one larger PR) by
-  the time of the application deadline.
-* The main criteria we use is quality of your best contributions, and
-  the bullets listed at
-  [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-  Because we focus on evaluating your best work, it doesn't hurt your
-  application to makes mistakes in your first few PRs as long as your
-  work improves.
-
-Zulip also participates in
-[Google Code-In](https://developers.google.com/open-source/gci/). Our
-selection criteria for Finalists and Grand Prize Winners is the same as our
-selection criteria for interns above.
-
-Most of our interns end up sticking around the project long-term, and many
-quickly become core team members. We hope you apply!
-
-### Google Summer of Code
-
-GSoC is by far the largest of our internship programs (14 students in
-2017; 11 in 2018; 17 in 2019).  While we don't control how many slots
-Google allocates to Zulip, we hope to mentor a similar number of
-students in future summers.
-
-If you're reading this well before the application deadline and want
-to make your application strong, we recommend getting involved in the
-community and fixing issues in Zulip now. Having good contributions
-and building a reputation for doing good work is best way to have a
-strong application.  About half of Zulip's GSoC students for Summer
-2017 had made significant contributions to the project by February
-2017, and about half had not.  Our
-[GSoC project ideas page][gsoc-guide] has lots more details on how
-Zulip does GSoC, as well as project ideas (though the project idea
-list is maintained only during the GSoC application period, so if
-you're looking at some other time of year, the project list is likely
-out-of-date).
-
-We also have in some past years run a Zulip Summer of Code (ZSoC)
-program for students who we didn't have enough slots to accept for
-GSoC but were able to find funding for.  Student expectations are the
-same as with GSoC, and it has no separate application process; your
-GSoC application is your ZSoC application.  If we'd like to select you
-for ZSoC, we'll contact you when the GSoC results are announced.
-
-[gsoc-guide]: https://zulip.readthedocs.io/en/latest/overview/gsoc-ideas.html
-[gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
-
-[1] Formally, [GSoC isn't an internship][gsoc-faq], but it is similar
-enough that we're treating it as such for the purposes of this
-documentation.
-
-## Zulip Outreach
-
-**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
-perception of projects like Zulip. We've collected a few sites below
-where we know Zulip has been discussed. Doing everything in the following
-list typically takes about 15 minutes.
-* Star us on GitHub. There are four main repositories:
-  [server/web](https://github.com/zulip/zulip),
-  [mobile](https://github.com/zulip/zulip-mobile),
-  [desktop](https://github.com/zulip/zulip-desktop), and
-  [Python API](https://github.com/zulip/python-zulip-api).
-* [Follow us](https://twitter.com/zulip) on Twitter.
-
-For both of the following, you'll need to make an account on the site if you
-don't already have one.
-
-* [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
-  AlternativeTo. We recommend upvoting a couple of other products you like
-  as well, both to give back to their community, and since single-upvote
-  accounts are generally given less weight. You can also
-  [upvote Zulip](https://alternativeto.net/software/slack/) on their page
-  for Slack.
-* [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
-  it, and upvote the reasons why people like Zulip that you find most
-  compelling. Again, we recommend adding a few other products that you like
-  as well.
-
-We have a doc with more detailed instructions and a few other sites, if you
-have been using Zulip for a while and want to contribute more.
-
-**Blog posts**. Writing a blog post about your experiences with Zulip, or
-about a technical aspect of Zulip can be a great way to spread the word
-about Zulip.
-
-We also occasionally [publish](http://blog.zulip.org/) longer form
-articles related to Zulip. Our posts typically get tens of thousands
-of views, and we always have good ideas for blog posts that we can
-outline but don't have time to write. If you are an experienced writer
-or copyeditor, send us a portfolio; we'd love to talk!
--- a/19
+++ b/19
@@ -0,0 +1,19 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN locale-gen en_US.UTF-8
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
+
+WORKDIR /srv/zulip
--- a/23
+++ b/23
@@ -1,23 +0,0 @@
-# To build run `docker build -f Dockerfile-postgresql .` from the root of the
-# zulip repo.
-
-# Currently the postgres images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image can not currently be upgraded
-# without users needing a manual pgdump and restore.
-
-# Install hunspell, zulip stop words, and run zulip database
-# init.
-FROM postgres:10
-COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop
-COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
-COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql
-COPY scripts/setup/pgroonga-debian.asc /tmp
-RUN apt-key add /tmp/pgroonga-debian.asc \
-    && echo "deb http://packages.groonga.org/debian/ stretch main" > /etc/apt/sources.list.d/zulip.list \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-       hunspell-en-us \
-       postgresql-${PG_MAJOR}-pgroonga \
-    && ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \
-    && ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \
-    && rm -rf /var/lib/apt/lists/*
--- a/1
+++ b/1
@@ -1,4 +1,3 @@
-Copyright 2011-2018 Dropbox, Inc., Kandra Labs, Inc., and contributors

                                 Apache License
                           Version 2.0, January 2004
--- a/16
+++ b/16
@@ -1,16 +0,0 @@
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this project except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-The software includes some works released by third parties under other
-free and open source licenses. Those works are redistributed under the
-license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.
--- a/README.md
+++ b/README.md
@@ -1,82 +1,263 @@
+**[Zulip overview](#zulip-overview)** |
+**[Community](#community)** |
+**[Installing for dev](#installing-the-zulip-development-environment)** |
+**[Installing for production](#running-zulip-in-production)** |
+**[Ways to contribute](#ways-to-contribute)** |
+**[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
+**[License](#license)**
+
 # Zulip overview

-Zulip is a powerful, open source group chat application that combines the
-immediacy of real-time chat with the productivity benefits of threaded
-conversations. Zulip is used by open source projects, Fortune 500 companies,
-large standards bodies, and others who need a real-time chat system that
-allows users to easily process hundreds or thousands of messages a day. With
-over 500 contributors merging over 500 commits a month, Zulip is also the
-largest and fastest growing open source group chat project.
+Zulip is a powerful, open source group chat application. Written in
+Python and using the Django framework, Zulip supports both private
+messaging and group chats via conversation streams.

-[![CircleCI branch](https://img.shields.io/circleci/project/github/zulip/zulip/master.svg)](https://circleci.com/gh/zulip/zulip/tree/master)
-[![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip/master.svg)](https://codecov.io/gh/zulip/zulip/branch/master)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
-[![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
+Zulip also supports fast search, drag-and-drop file uploads, image
+previews, group private messages, audible notifications,
+missed-message emails, desktop apps, and much more.

-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
+Further information on the Zulip project and its features can be found
+at https://www.zulip.org.

-## Getting started
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master) [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/) [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)

-Click on the appropriate link below. If nothing seems to apply,
-join us on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-and tell us what's up!
+## Community

-You might be interested in:
+There are several places online where folks discuss Zulip.

-* **Contributing code**. Check out our
-  [guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
-  to get started. Zulip prides itself on maintaining a clean and
-  well-tested codebase, and a stock of hundreds of
-  [beginner-friendly issues][beginner-friendly].
+One of those places is our [public Zulip instance](https://chat.zulip.org/).
+You can go through the simple signup process at that link, and then you
+will soon be talking to core Zulip developers and other users.  To get
+help in real time, you will have the best luck finding core developers
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get a reply
+within minutes to a few hours, depending on time of day.

-* **Contributing non-code**.
-  [Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issue),
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
-  into your language,
-  [write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
-  for the Zulip blog, or
-  [give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
-  would love to hear from you, even if you're just trying the product out.
+For Google Summer of Code students and applicants, we have [a mailing
+list](https://groups.google.com/forum/#!forum/zulip-gsoc) for help,
+questions, and announcements.

-* **Supporting Zulip**. Advocate for your organization to use Zulip, write a
-  review in the mobile app stores, or
-  [upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
-  product comparison sites.
+We have
+[a public mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
+that is currently pretty low traffic because most discussions happen
+in our public Zulip instance.  We use it to announce Zulip developer
+community gatherings and ask for feedback on major technical or design
+decisions.  It has several hundred subscribers, so you can use it to
+ask questions about features or possible bugs, but please don't use it
+ask for generic help getting started as a contributor (e.g. because
+you want to do Google Summer of Code).  The rest of this page covers
+how to get involved in the Zulip project in detail.

-* **Checking Zulip out**. The best way to see Zulip in action is to drop by
-  the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
-  also recommend reading Zulip for
-  [open source](https://zulipchat.com/for/open-source/), Zulip for
-  [companies](https://zulipchat.com/for/companies/), or Zulip for
-  [working groups and part time communities](https://zulipchat.com/for/working-groups-and-communities/).
+Zulip also has a [blog](https://blog.zulip.org/).

-* **Running a Zulip server**. Use a preconfigured [Digital Ocean droplet](https://marketplace.digitalocean.com/apps/zulip),
-  [install Zulip](https://zulip.readthedocs.io/en/stable/production/install.html)
-  directly, or use Zulip's
-  experimental [Docker image](https://zulip.readthedocs.io/en/latest/production/deployment.html#zulip-in-docker).
-  Commercial support is available; see <https://zulipchat.com/plans> for details.
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
+track Zulip-related issues (and store our code, of course).
+Anybody with a GitHub account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull
+Requests as our primary mechanism to receive code contributions.

-* **Using Zulip without setting up a server**. <https://zulipchat.com> offers
-  free and commercial hosting.
+The Zulip community has a [Code of Conduct][code-of-conduct].

-* **Applying for a Zulip internship**. Zulip runs internship programs with
-  [Outreachy](https://www.outreachy.org/),
-  [Google Summer of Code](https://developers.google.com/open-source/gsoc/),
-  and the
-  [MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram). Zulip
-  also participates in
-  [Google Code-In](https://developers.google.com/open-source/gci/). More
-  information is available
-  [here](https://zulip.readthedocs.io/en/latest/overview/contributing.html#internship-programs).
+## Installing the Zulip Development environment

-You may also be interested in reading our [blog](http://blog.zulip.org/) or
-following us on [twitter](https://twitter.com/zulip).
-Zulip is distributed under the
-[Apache 2.0](https://github.com/zulip/zulip/blob/master/LICENSE) license.
+The Zulip development environment is the recommended option for folks
+interested in trying out Zulip.  This is documented in [the developer
+installation guide][dev-install].

-[beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22
+## Running Zulip in production
+
+Zulip in production supports Ubuntu 14.04 Trusty and Ubuntu 16.04
+Xenial. Work is ongoing on adding support for additional
+platforms. The installation process is documented at
+https://zulip.org/server.html and in more detail in [the
+documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
+
+## Ways to contribute
+
+Zulip welcomes all forms of contributions!  The page documents the
+Zulip development process.
+
+* **Pull requests**. Before a pull request can be merged, you need to
+sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
+
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+docs][doc-test]. You can also try out [our new desktop
+client][electron], which is in alpha; we'd appreciate testing and
+[feedback](https://github.com/zulip/zulip-electron/issues/new).
+
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat]. You can also improve
+[Zulip.org][z-org].
+
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list](#community) and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list!  Please report any security issues you discover to
+zulip-security@googlegroups.com.
+
+* **App codebases**. This repository is for the Zulip server and web
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].
+
+* **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
+integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
+and [Trello][]), plus [node.js API bindings][node], an [isomorphic
+ JavaScript library][zulip-js], and a [full-text search PostgreSQL
+ extension][tsearch], as separate repos.
+
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation][transifex] if you're interested in
+contributing!
+
+[cla]: https://opensource.dropbox.com/cla/
+[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
+[dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
+[electron]: https://github.com/zulip/zulip-electron
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
+[ios-exp]: https://github.com/zulip/zulip-mobile
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[hubot-adapter]: https://github.com/zulip/hubot-zulip
+[jenkins]: https://github.com/zulip/zulip-jenkins-plugin
+[node]: https://github.com/zulip/zulip-node
+[zulip-js]: https://github.com/zulip/zulip-js
+[phab]: https://github.com/zulip/phabricator-to-zulip
+[puppet]: https://github.com/matthewbarr/puppet-zulip
+[redmine]: https://github.com/zulip/zulip-redmine-plugin
+[trello]: https://github.com/zulip/trello-to-zulip
+[tsearch]: https://github.com/zulip/tsearch_extras
+[transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
+[z-org]: https://github.com/zulip/zulip.github.io
+
+## Google Summer of Code
+
+We participated in
+[GSoC](https://developers.google.com/open-source/gsoc/) last year and
+hope to do so again in 2017.  For guidance, please read
+[our GSoC instructions and ideas page](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
+and feel free to email
+[our GSoC mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc).
+
+## How to get involved with contributing to Zulip
+
+First, subscribe to the Zulip [development discussion mailing
+list](#community).
+
+The Zulip project uses a system of labels in our [issue
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
+
+* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+  The Zulip project loves contributions of new documentation.
+
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
+  The projects that are
+  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
+  These are great projects if you're looking to make an impact.
+
+Another way to find issues in Zulip is to take advantage of our
+"area:<foo>" convention in separating out issues.  We partition all of
+our issues into areas like admin, compose, emoji, hotkeys, i18n,
+onboarding, search, etc.  You can see this here:
+
+[https://github.com/zulip/zulip/labels]
+
+Click on any of the "area:" labels and you will see all the tickets
+related to your area of interest.
+
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.
+
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
+
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
+
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.
+
+Finally, before implementing a larger feature, we highly recommend
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.
+
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just send an email
+to the Zulip Developers list with your thoughts.
+
+## License
+
+Copyright 2011-2016 Dropbox, Inc. and contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The software includes some works released by third parties under other
+free and open source licenses. Those works are redistributed under the
+license terms under which the works were received. For more details,
+see the ``docs/THIRDPARTY`` file included with this distribution.
--- a/149
+++ b/149
@@ -10,7 +10,7 @@ end
 if Vagrant::VERSION == "1.8.7" then
    path = `which curl`
    if path.include?('/opt/vagrant/embedded/bin/curl') then
-        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 "\
+        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
             "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
             "issue before provisioning. See "\
             "https://github.com/mitchellh/vagrant/issues/7997 "\
@@ -19,46 +19,16 @@ if Vagrant::VERSION == "1.8.7" then
    end
 end

-# Workaround: Vagrant removed the atlas.hashicorp.com to
-# vagrantcloud.com redirect in February 2018. The value of
-# DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
-# atlas.hashicorp.com, which means that removal broke the fetching and
-# updating of boxes (since the old URL doesn't work).  See
-# https://github.com/hashicorp/vagrant/issues/9442
-if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
-  Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
-end
-
-# Monkey patch https://github.com/hashicorp/vagrant/pull/10879 so we
-# can fall back to another provider if docker is not installed.
-begin
-  require Vagrant.source_root.join("plugins", "providers", "docker", "provider")
-rescue LoadError
-else
-  VagrantPlugins::DockerProvider::Provider.class_eval do
-    method(:usable?).owner == singleton_class or def self.usable?(raise_error=false)
-      VagrantPlugins::DockerProvider::Driver.new.execute("docker", "version")
-      true
-    rescue Vagrant::Errors::CommandUnavailable, VagrantPlugins::DockerProvider::Errors::ExecuteError
-      raise if raise_error
-      return false
-    end
-  end
-end
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

+  # For LXC. VirtualBox hosts use a different box, described below.
+  config.vm.box = "fgrehm/trusty64-lxc"
+
  # The Zulip development environment runs on 9991 on the guest.
  host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
+  http_proxy = https_proxy = no_proxy = ""
  host_ip_addr = "127.0.0.1"

-  # System settings for the virtual machine.
-  vm_num_cpus = "2"
-  vm_memory = "2048"
-
-  ubuntu_mirror = ""
-
  config.vm.synced_folder ".", "/vagrant", disabled: true
  config.vm.synced_folder ".", "/srv/zulip"

@@ -74,118 +44,63 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
      when "NO_PROXY"; no_proxy = value
      when "HOST_PORT"; host_port = value.to_i
      when "HOST_IP_ADDR"; host_ip_addr = value
-      when "GUEST_CPUS"; vm_num_cpus = value
-      when "GUEST_MEMORY_MB"; vm_memory = value
-      when "UBUNTU_MIRROR"; ubuntu_mirror = value
      end
    end
  end

+  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
+
  if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
+    if http_proxy != ""
      config.proxy.http = http_proxy
    end
-    if !https_proxy.nil?
+    if https_proxy != ""
      config.proxy.https = https_proxy
    end
-    if !no_proxy.nil?
+    if https_proxy != ""
      config.proxy.no_proxy = no_proxy
    end
-  elsif !http_proxy.nil? or !https_proxy.nil?
-    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
-    # We haven't figured out a workaround.
-    puts 'You have specified value for proxy in ~/.zulip-vagrant-config file but did not ' \
-         'install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install ' \
-         'vagrant-proxyconf` in a terminal.  This error will appear twice.'
-    exit
  end

-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
-  # Specify Docker provider before VirtualBox provider so it's preferred.
-  config.vm.provider "docker" do |d, override|
-    d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
-    d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
-    if !ubuntu_mirror.empty?
-      d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
+  # Specify LXC provider before VirtualBox provider so it's preferred.
+  config.vm.provider "lxc" do |lxc|
+    if command? "lxc-ls"
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
+      if LXC_VERSION >= "1.1.0"
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
+        lxc.customize 'aa_allow_incomplete', 1
+      end
+      if LXC_VERSION >= "2.0.0"
+        lxc.backingstore = 'dir'
+      end
    end
-    d.has_ssh = true
-    d.create_args = ["--ulimit", "nofile=1024:65536"]
  end

  config.vm.provider "virtualbox" do |vb, override|
-    override.vm.box = "ubuntu/bionic64"
-    # An unnecessary log file gets generated when running vagrant up for the
-    # first time with the Ubuntu Bionic box. This looks like it is being
-    # caused upstream by the base box containing a Vagrantfile with a similar
-    # line to the one below.
-    # see https://github.com/hashicorp/vagrant/issues/9425
-    vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ]
+    override.vm.box = "ubuntu/trusty64"
    # It's possible we can get away with just 1.5GB; more testing needed
-    vb.memory = vm_memory
-    vb.cpus = vm_num_cpus
+    vb.memory = 2048
+    vb.cpus = 2
+  end
+
+  config.vm.provider "vmware_fusion" do |vb, override|
+    override.vm.box = "puphpet/ubuntu1404-x64"
+    vb.vmx["memsize"] = "2048"
+    vb.vmx["numvcpus"] = "2"
  end

 $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
-
-# Code should go here, rather than tools/provision, only if it is
-# something that we don't want to happen when running provision in a
-# development environment not using Vagrant.
-
-# Set the Ubuntu mirror
-[ ! '#{ubuntu_mirror}' ] || sudo sed -i 's|http://\\(\\w*\\.\\)*archive\\.ubuntu\\.com/ubuntu/\\? |#{ubuntu_mirror} |' /etc/apt/sources.list
-
-# Set the MOTD on the system to have Zulip instructions
-sudo ln -nsf /srv/zulip/tools/setup/dev-motd /etc/update-motd.d/99-zulip-dev
-sudo rm -f /etc/update-motd.d/10-help-text
-sudo dpkg --purge landscape-client landscape-common ubuntu-release-upgrader-core update-manager-core update-notifier-common ubuntu-server
-sudo dpkg-divert --add --rename /etc/default/motd-news
-sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
-
 # If the host is running SELinux remount the /sys/fs/selinux directory as read only,
 # needed for apt-get to work.
 if [ -d "/sys/fs/selinux" ]; then
-    sudo mount -o remount,ro /sys/fs/selinux
+  sudo mount -o remount,ro /sys/fs/selinux
 fi
-
-# Set default locale, this prevents errors if the user has another locale set.
-if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
-    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
-fi
-
-# Set an environment variable, so that we won't print the virtualenv
-# shell warning (it'll be wrong, since the shell is dying anyway)
-export SKIP_VENV_SHELL_WARNING=1
-
-# End `set -x`, so that the end of provision doesn't look like an error
-# message after a successful run.
-set +x
-
-# Check if the zulip directory is writable
-if [ ! -w /srv/zulip ]; then
-    echo "The vagrant user is unable to write to the zulip directory."
-    echo "To fix this, run the following commands on the host machine:"
-    # sudo is required since our uid is not 1000
-    echo '    vagrant halt -f'
-    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
-    echo '    sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
-    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
-    echo "You can resume setting up your vagrant environment by running:"
-    echo "    vagrant up"
-    exit 1
-fi
-# Provision the development environment
 ln -nsf /srv/zulip ~/zulip
 /srv/zulip/tools/provision
-
-# Run any custom provision hooks the user has configured
-if [ -f /srv/zulip/tools/custom_provision ]; then
-    chmod +x /srv/zulip/tools/custom_provision
-    /srv/zulip/tools/custom_provision
-fi
 SCRIPT

  config.vm.provision "shell",
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -1,86 +1,71 @@
-import time
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
-import logging
-from typing import Callable, Dict, List, \
-    Optional, Tuple, Type, Union
-
+from django.db import connection, models
+from django.utils import timezone
 from django.conf import settings
-from django.db import connection
-from django.db.models import F
+from datetime import timedelta, datetime

-from analytics.models import BaseCount, \
-    FillState, InstallationCount, RealmCount, StreamCount, \
-    UserCount, installation_epoch, last_successful_fill
-from zerver.lib.logging_util import log_to_file
-from zerver.lib.timestamp import ceiling_to_day, \
-    ceiling_to_hour, floor_to_hour, verify_UTC
-from zerver.models import Message, Realm, RealmAuditLog, \
-    Stream, UserActivityInterval, UserProfile, models
+from analytics.models import InstallationCount, RealmCount, \
+    UserCount, StreamCount, BaseCount, FillState, installation_epoch
+from zerver.models import Realm, UserProfile, Message, Stream, models
+from zerver.lib.timestamp import floor_to_day
+
+from typing import Any, Optional, Type, Tuple, Text
+
+import logging
+import time

 ## Logging setup ##
+log_format = '%(asctime)s %(levelname)-8s %(message)s'
+logging.basicConfig(format=log_format)

-logger = logging.getLogger('zulip.management')
-log_to_file(logger, settings.ANALYTICS_LOG_PATH)
+formatter = logging.Formatter(log_format)
+file_handler = logging.FileHandler(settings.ANALYTICS_LOG_PATH)
+file_handler.setFormatter(formatter)

-# You can't subtract timedelta.max from a datetime, so use this instead
-TIMEDELTA_MAX = timedelta(days=365*1000)
+logger = logging.getLogger("zulip.management")
+logger.setLevel(logging.INFO)
+logger.addHandler(file_handler)

-## Class definitions ##
+# First post office in Boston
+MIN_TIME = datetime(1639, 1, 1, 0, 0, 0, tzinfo=timezone.utc)

-class CountStat:
+class CountStat(object):
    HOUR = 'hour'
    DAY = 'day'
    FREQUENCIES = frozenset([HOUR, DAY])
+    # Allowed intervals are HOUR, DAY, and, GAUGE
+    GAUGE = 'gauge'

-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta]=None) -> None:
+    def __init__(self, property, zerver_count_query, filter_args, group_by, frequency, is_gauge):
+        # type: (str, ZerverCountQuery, Dict[str, bool], Optional[Tuple[models.Model, str]], str, bool) -> None
        self.property = property
-        self.data_collector = data_collector
+        self.zerver_count_query = zerver_count_query
        # might have to do something different for bitfields
+        self.filter_args = filter_args
+        self.group_by = group_by
        if frequency not in self.FREQUENCIES:
-            raise AssertionError("Unknown frequency: %s" % (frequency,))
+            raise ValueError("Unknown frequency: %s" % (frequency,))
        self.frequency = frequency
-        if interval is not None:
-            self.interval = interval
-        elif frequency == CountStat.HOUR:
-            self.interval = timedelta(hours=1)
-        else:  # frequency == CountStat.DAY
-            self.interval = timedelta(days=1)
+        self.interval = self.GAUGE if is_gauge else frequency

-    def __str__(self) -> str:
-        return "<CountStat: %s>" % (self.property,)
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<CountStat: %s>" % (self.property,)

-class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
-        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
+class ZerverCountQuery(object):
+    def __init__(self, zerver_table, analytics_table, query):
+        # type: (Type[models.Model], Type[BaseCount], Text) -> None
+        self.zerver_table = zerver_table
+        self.analytics_table = analytics_table
+        self.query = query

-class DependentCountStat(CountStat):
-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta]=None, dependencies: List[str]=[]) -> None:
-        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
-        self.dependencies = dependencies
-
-class DataCollector:
-    def __init__(self, output_table: Type[BaseCount],
-                 pull_function: Optional[Callable[[str, datetime, datetime], int]]) -> None:
-        self.output_table = output_table
-        self.pull_function = pull_function
-
-## CountStat-level operations ##
-
-def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
-    if stat.frequency == CountStat.HOUR:
-        time_increment = timedelta(hours=1)
-    elif stat.frequency == CountStat.DAY:
-        time_increment = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
-
-    verify_UTC(fill_to_time)
-    if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
+def do_update_fill_state(fill_state, end_time, state):
+    # type: (FillState, datetime, int) -> None
+    fill_state.end_time = end_time
+    fill_state.state = state
+    fill_state.save()

+def process_count_stat(stat, fill_to_time):
+    # type: (CountStat, datetime) -> None
    fill_state = FillState.objects.filter(property=stat.property).first()
    if fill_state is None:
        currently_filled = installation_epoch()
@@ -90,90 +75,86 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
        logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
    elif fill_state.state == FillState.STARTED:
        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
-        do_delete_counts_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - time_increment
+        do_delete_count_stat_at_hour(stat, fill_state.end_time)
+        currently_filled = fill_state.end_time - timedelta(hours = 1)
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        logger.info("UNDO DONE %s" % (stat.property,))
    elif fill_state.state == FillState.DONE:
        currently_filled = fill_state.end_time
    else:
-        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
+        raise ValueError("Unknown value for FillState.state: %s." % (fill_state.state,))

-    if isinstance(stat, DependentCountStat):
-        for dependency in stat.dependencies:
-            dependency_fill_time = last_successful_fill(dependency)
-            if dependency_fill_time is None:
-                logger.warning("DependentCountStat %s run before dependency %s." %
-                               (stat.property, dependency))
-                return
-            fill_to_time = min(fill_to_time, dependency_fill_time)
-
-    currently_filled = currently_filled + time_increment
+    currently_filled = currently_filled + timedelta(hours = 1)
    while currently_filled <= fill_to_time:
-        logger.info("START %s %s" % (stat.property, currently_filled))
+        logger.info("START %s %s %s" % (stat.property, stat.interval, currently_filled))
        start = time.time()
        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
        do_fill_count_stat_at_hour(stat, currently_filled)
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        end = time.time()
-        currently_filled = currently_filled + time_increment
-        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
+        currently_filled = currently_filled + timedelta(hours = 1)
+        logger.info("DONE %s %s (%dms)" % (stat.property, stat.interval, (end-start)*1000))

-def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
+# We assume end_time is on an hour boundary, and is timezone aware.
+# It is the caller's responsibility to enforce this!
+def do_fill_count_stat_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
+    if stat.frequency == CountStat.DAY and (end_time != floor_to_day(end_time)):
+        return

-# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is timezone aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(stat: CountStat, end_time: datetime) -> None:
-    start_time = end_time - stat.interval
-    if not isinstance(stat, LoggingCountStat):
-        timer = time.time()
-        assert(stat.data_collector.pull_function is not None)
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
-        logger.info("%s run pull_function (%dms/%sr)" %
-                    (stat.property, (time.time()-timer)*1000, rows_added))
+    if stat.interval == CountStat.HOUR:
+        start_time = end_time - timedelta(hours = 1)
+    elif stat.interval == CountStat.DAY:
+        start_time = end_time - timedelta(days = 1)
+    else: # stat.interval == CountStat.GAUGE
+        start_time = MIN_TIME
+
+    do_pull_from_zerver(stat, start_time, end_time)
    do_aggregate_to_summary_table(stat, end_time)

-def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
-    if isinstance(stat, LoggingCountStat):
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.output_table in [UserCount, StreamCount]:
-            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-    else:
-        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
+def do_delete_count_stat_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
+    UserCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    StreamCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    RealmCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    InstallationCount.objects.filter(property = stat.property, end_time = end_time).delete()

-def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
+def do_drop_all_analytics_tables():
+    # type: () -> None
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+
+def do_aggregate_to_summary_table(stat, end_time):
+    # type: (CountStat, datetime) -> None
    cursor = connection.cursor()

    # Aggregate into RealmCount
-    output_table = stat.data_collector.output_table
-    if output_table in (UserCount, StreamCount):
+    analytics_table = stat.zerver_count_query.analytics_table
+    if analytics_table in (UserCount, StreamCount):
        realmcount_query = """
            INSERT INTO analytics_realmcount
                (realm_id, value, property, subgroup, end_time)
            SELECT
-                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
-                %(output_table)s.subgroup, %%(end_time)s
+                zerver_realm.id, COALESCE(sum(%(analytics_table)s.value), 0), '%(property)s',
+                %(analytics_table)s.subgroup, %%(end_time)s
            FROM zerver_realm
-            JOIN %(output_table)s
+            JOIN %(analytics_table)s
            ON
-                zerver_realm.id = %(output_table)s.realm_id
-            WHERE
-                %(output_table)s.property = '%(property)s' AND
-                %(output_table)s.end_time = %%(end_time)s
-            GROUP BY zerver_realm.id, %(output_table)s.subgroup
-        """ % {'output_table': output_table._meta.db_table,
+            (
+                %(analytics_table)s.realm_id = zerver_realm.id AND
+                %(analytics_table)s.property = '%(property)s' AND
+                %(analytics_table)s.end_time = %%(end_time)s
+            )
+            GROUP BY zerver_realm.id, %(analytics_table)s.subgroup
+        """ % {'analytics_table': analytics_table._meta.db_table,
               'property': stat.property}
        start = time.time()
        cursor.execute(realmcount_query, {'end_time': end_time})
        end = time.time()
-        logger.info("%s RealmCount aggregation (%dms/%sr)" % (
-            stat.property, (end - start) * 1000, cursor.rowcount))
+        logger.info("%s RealmCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))

    # Aggregate into InstallationCount
    installationcount_query = """
@@ -183,276 +164,85 @@ def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
            sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
        FROM analytics_realmcount
        WHERE
+        (
            property = '%(property)s' AND
            end_time = %%(end_time)s
-        GROUP BY analytics_realmcount.subgroup
+        ) GROUP BY analytics_realmcount.subgroup
    """ % {'property': stat.property}
    start = time.time()
    cursor.execute(installationcount_query, {'end_time': end_time})
    end = time.time()
-    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (
-        stat.property, (end - start) * 1000, cursor.rowcount))
+    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
    cursor.close()

-## Utility functions called from outside counts.py ##
-
-# called from zerver/lib/actions.py; should not throw any errors
-def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream], stat: CountStat,
-                              subgroup: Optional[Union[str, int, bool]], event_time: datetime,
-                              increment: int=1) -> None:
-    table = stat.data_collector.output_table
-    if table == RealmCount:
-        id_args = {'realm': zerver_object}
-    elif table == UserCount:
-        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
-    else:  # StreamCount
-        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
-
-    if stat.frequency == CountStat.DAY:
-        end_time = ceiling_to_day(event_time)
-    else:  # CountStat.HOUR:
-        end_time = ceiling_to_hour(event_time)
-
-    row, created = table.objects.get_or_create(
-        property=stat.property, subgroup=subgroup, end_time=end_time,
-        defaults={'value': increment}, **id_args)
-    if not created:
-        row.value = F('value') + increment
-        row.save(update_fields=['value'])
-
-def do_drop_all_analytics_tables() -> None:
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-def do_drop_single_stat(property: str) -> None:
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-## DataCollector-level operations ##
-
-def do_pull_by_sql_query(property: str, start_time: datetime, end_time: datetime, query: str,
-                         group_by: Optional[Tuple[models.Model, str]]) -> int:
-    if group_by is None:
+# This is the only method that hits the prod databases directly.
+def do_pull_from_zerver(stat, start_time, end_time):
+    # type: (CountStat, datetime, datetime) -> None
+    zerver_table = stat.zerver_count_query.zerver_table._meta.db_table # type: ignore
+    join_args = ' '.join('AND %s.%s = %s' % (zerver_table, key, value)
+                         for key, value in stat.filter_args.items())
+    if stat.group_by is None:
        subgroup = 'NULL'
        group_by_clause  = ''
    else:
-        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
+        subgroup = '%s.%s' % (stat.group_by[0]._meta.db_table, stat.group_by[1])
        group_by_clause = ', ' + subgroup

-    # We do string replacement here because cursor.execute will reject a
-    # group_by_clause given as a param.
-    # We pass in the datetimes as params to cursor.execute so that we don't have to
-    # think about how to convert python datetimes to SQL datetimes.
-    query_ = query % {'property': property, 'subgroup': subgroup,
-                      'group_by_clause': group_by_clause}
+    # We do string replacement here because passing join_args as a param
+    # may result in problems when running cursor.execute; we do
+    # the string formatting prior so that cursor.execute runs it as sql
+    query_ = stat.zerver_count_query.query % {'zerver_table': zerver_table,
+                                              'property': stat.property,
+                                              'join_args': join_args,
+                                              'subgroup': subgroup,
+                                              'group_by_clause': group_by_clause}
    cursor = connection.cursor()
+    start = time.time()
    cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
-    rowcount = cursor.rowcount
+    end = time.time()
+    logger.info("%s do_pull_from_zerver (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
    cursor.close()
-    return rowcount

-def sql_data_collector(output_table: Type[BaseCount], query: str,
-                       group_by: Optional[Tuple[models.Model, str]]) -> DataCollector:
-    def pull_function(property: str, start_time: datetime, end_time: datetime) -> int:
-        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
-    return DataCollector(output_table, pull_function)
-
-def do_pull_minutes_active(property: str, start_time: datetime, end_time: datetime) -> int:
-    user_activity_intervals = UserActivityInterval.objects.filter(
-        end__gt=start_time, start__lt=end_time
-    ).select_related(
-        'user_profile'
-    ).values_list(
-        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
-
-    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
-    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        start = max(start_time, interval_start)
-        end = min(end_time, interval_end)
-        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
-
-    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
-                      end_time=end_time, value=int(seconds // 60))
-            for ids, seconds in seconds_active.items() if seconds >= 60]
-    UserCount.objects.bulk_create(rows)
-    return len(rows)
-
-count_message_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
-        '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_message
-    ON
-        zerver_userprofile.id = zerver_message.sender_id
-    WHERE
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_message.date_sent >= %%(time_start)s AND
-        zerver_message.date_sent < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-# Note: ignores the group_by / group_by_clause.
-count_message_type_by_user_query = """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type = 1 THEN 'private_message'
-             WHEN
-                  zerver_recipient.type = 3 THEN 'huddle_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
-
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.date_sent >= %%(time_start)s AND
-            zerver_message.date_sent < %%(time_end)s
-        JOIN zerver_recipient
-        ON
-            zerver_message.recipient_id = zerver_recipient.id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_recipient.type_id = zerver_stream.id
-        GROUP BY
-            zerver_userprofile.realm_id, zerver_userprofile.id,
-            zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-
-# This query joins to the UserProfile table since all current queries that
-# use this also subgroup on UserProfile.is_bot. If in the future there is a
-# stat that counts messages by stream and doesn't need the UserProfile
-# table, consider writing a new query for efficiency.
-count_message_by_stream_query = """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-        zerver_stream.id = zerver_recipient.type_id
-    JOIN zerver_message
-    ON
-        zerver_recipient.id = zerver_message.recipient_id
-    JOIN zerver_userprofile
-    ON
-        zerver_message.sender_id = zerver_userprofile.id
-    WHERE
-        zerver_stream.date_created < %%(time_end)s AND
-        zerver_recipient.type = 2 AND
-        zerver_message.date_sent >= %%(time_start)s AND
-        zerver_message.date_sent < %%(time_end)s
-    GROUP BY zerver_stream.id %(group_by_clause)s
-"""
-
-# Hardcodes the query needed by active_users:is_bot:day, since that is
-# currently the only stat that uses this.
 count_user_by_realm_query = """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
-        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
+        zerver_realm.id, count(%(zerver_table)s),'%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_realm
    JOIN zerver_userprofile
    ON
-        zerver_realm.id = zerver_userprofile.realm_id
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
+    (
+        zerver_userprofile.realm_id = zerver_realm.id AND
        zerver_userprofile.date_joined >= %%(time_start)s AND
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_userprofile.is_active = TRUE
+        zerver_userprofile.date_joined < %%(time_end)s
+        %(join_args)s
+    )
+    WHERE
+        zerver_realm.date_created < %%(time_end)s
    GROUP BY zerver_realm.id %(group_by_clause)s
 """
+zerver_count_user_by_realm = ZerverCountQuery(UserProfile, RealmCount, count_user_by_realm_query)

-# Currently hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
-# In particular, it's important to ensure that migrations don't cause that to happen.
-check_realmauditlog_by_user_query = """
+# currently .sender_id is only Message specific thing
+count_message_by_user_query = """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realmauditlog ral1
-    JOIN (
-        SELECT modified_user_id, max(event_time) AS max_event_time
-        FROM zerver_realmauditlog
-        WHERE
-            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
-            event_time < %%(time_end)s
-        GROUP BY modified_user_id
-    ) ral2
-    ON
-        ral1.event_time = max_event_time AND
-        ral1.modified_user_id = ral2.modified_user_id
-    JOIN zerver_userprofile
-    ON
-        ral1.modified_user_id = zerver_userprofile.id
-    WHERE
-        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
-""".format(user_created=RealmAuditLog.USER_CREATED,
-           user_activated=RealmAuditLog.USER_ACTIVATED,
-           user_deactivated=RealmAuditLog.USER_DEACTIVATED,
-           user_reactivated=RealmAuditLog.USER_REACTIVATED)
-
-check_useractivityinterval_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
+        zerver_userprofile.id, zerver_userprofile.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_userprofile
-    JOIN zerver_useractivityinterval
+    JOIN zerver_message
    ON
-        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
+    (
+        zerver_message.sender_id = zerver_userprofile.id AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s
+        %(join_args)s
+    )
    WHERE
-        zerver_useractivityinterval.end >= %%(time_start)s AND
-        zerver_useractivityinterval.start < %%(time_end)s
+            zerver_userprofile.date_joined < %%(time_end)s
    GROUP BY zerver_userprofile.id %(group_by_clause)s
 """
-
-count_realm_active_humans_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
-    FROM (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = 'active_users_audit:is_bot:day' AND
-            subgroup = 'false' AND
-            end_time = %%(time_end)s
-    ) usercount1
-    JOIN (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = '15day_actives::day' AND
-            end_time = %%(time_end)s
-    ) usercount2
-    ON
-        usercount1.user_id = usercount2.user_id
-    GROUP BY usercount1.realm_id
-"""
+zerver_count_message_by_user = ZerverCountQuery(Message, UserCount, count_message_by_user_query)

 # Currently unused and untested
 count_stream_by_realm_query = """
@@ -463,79 +253,101 @@ count_stream_by_realm_query = """
    FROM zerver_realm
    JOIN zerver_stream
    ON
-        zerver_realm.id = zerver_stream.realm_id AND
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
+    (
+        zerver_stream.realm_id = zerver_realm.id AND
        zerver_stream.date_created >= %%(time_start)s AND
        zerver_stream.date_created < %%(time_end)s
+        %(join_args)s
+    )
+    WHERE
+        zerver_realm.date_created < %%(time_end)s
    GROUP BY zerver_realm.id %(group_by_clause)s
 """
+zerver_count_stream_by_realm = ZerverCountQuery(Stream, RealmCount, count_stream_by_realm_query)

-## CountStat declarations ##
+# This query violates the count_X_by_Y_query conventions in several ways. One,
+# the X table is not specified by the query name; MessageType is not a zerver
+# table. Two, it ignores the subgroup column in the CountStat object; instead,
+# it uses 'message_type' from the subquery to fill in the subgroup column.
+count_message_type_by_user_query = """
+    INSERT INTO analytics_usercount
+            (realm_id, user_id, value, property, subgroup, end_time)
+    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
+    FROM
+    (
+        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
+        CASE WHEN
+                  zerver_recipient.type != 2 THEN 'private_message'
+             WHEN
+                  zerver_stream.invite_only = TRUE THEN 'private_stream'
+             ELSE 'public_stream'
+        END
+        message_type

-count_stats_ = [
-    # Messages Sent stats
-    # Stats that count the number of messages sent in various ways.
-    # These are also the set of stats that read from the Message table.
+        FROM zerver_userprofile
+        JOIN zerver_message
+        ON
+            zerver_message.sender_id = zerver_userprofile.id AND
+            zerver_message.pub_date >= %%(time_start)s AND
+            zerver_message.pub_date < %%(time_end)s
+            %(join_args)s
+        JOIN zerver_recipient
+        ON
+            zerver_recipient.id = zerver_message.recipient_id
+        LEFT JOIN zerver_stream
+        ON
+            zerver_stream.id = zerver_recipient.type_id
+        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
+    ) AS subquery
+    GROUP BY realm_id, id, message_type
+"""
+zerver_count_message_type_by_user = ZerverCountQuery(Message, UserCount, count_message_type_by_user_query)

-    CountStat('messages_sent:is_bot:hour',
-              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.HOUR),
-    CountStat('messages_sent:message_type:day',
-              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
-    CountStat('messages_sent:client:day',
-              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
-              CountStat.DAY),
-    CountStat('messages_in_stream:is_bot:day',
-              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
+# Note that this query also joins to the UserProfile table, since all
+# current queries that use this also subgroup on UserProfile.is_bot. If in
+# the future there is a query that counts messages by stream and doesn't need
+# the UserProfile table, consider writing a new query for efficiency.
+count_message_by_stream_query = """
+    INSERT INTO analytics_streamcount
+        (stream_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_stream
+    JOIN zerver_recipient
+    ON
+    (
+        zerver_recipient.type = 2 AND
+        zerver_stream.id = zerver_recipient.type_id
+    )
+    JOIN zerver_message
+    ON
+    (
+        zerver_message.recipient_id = zerver_recipient.id AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s AND
+        zerver_stream.date_created < %%(time_end)s
+        %(join_args)s
+    )
+    JOIN zerver_userprofile
+    ON zerver_userprofile.id = zerver_message.sender_id
+    GROUP BY zerver_stream.id %(group_by_clause)s
+"""
+zerver_count_message_by_stream = ZerverCountQuery(Message, StreamCount, count_message_by_stream_query)

-    # Number of Users stats
-    # Stats that count the number of active users in the UserProfile.is_active sense.
-
-    # 'active_users_audit:is_bot:day' is the canonical record of which users were
-    # active on which days (in the UserProfile.is_active sense).
-    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-    CountStat('active_users_audit:is_bot:day',
-              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-    # sum sequence of 'active_users_log:is_bot:day', for any realm that
-    # started after the latter stat was introduced.
-    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
-    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-    # approximation, e.g. if a user is deactivated between the end of the
-    # day and when this stat is run, they won't be counted. However, is the
-    # simplest of the three to inspect by hand.
-    CountStat('active_users:is_bot:day',
-              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
-              CountStat.DAY, interval=TIMEDELTA_MAX),
-
-    # User Activity stats
-    # Stats that measure user activity in the UserActivityInterval sense.
-
-    CountStat('1day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=1)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('15day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
-
-    # Rate limiting stats
-
-    # Used to limit the number of invitation emails sent by a realm
-    LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
-
-    # Dependent stats
-    # Must come after their dependencies.
-
-    # Canonical account of the number of active humans in a realm on each day.
-    DependentCountStat('realm_active_humans::day',
-                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
-                       CountStat.DAY,
-                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
-]
-
-COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
+COUNT_STATS = {
+    'active_users:is_bot:day': CountStat(
+        'active_users:is_bot:day', zerver_count_user_by_realm, {'is_active': True},
+        (UserProfile, 'is_bot'), CountStat.DAY, True),
+    'messages_sent:is_bot:hour': CountStat(
+        'messages_sent:is_bot:hour', zerver_count_message_by_user, {},
+        (UserProfile, 'is_bot'), CountStat.HOUR, False),
+    'messages_sent:message_type:day': CountStat(
+        'messages_sent:message_type:day', zerver_count_message_type_by_user, {},
+        None, CountStat.DAY, False),
+    'messages_sent:client:day': CountStat(
+        'messages_sent:client:day', zerver_count_message_by_user, {},
+        (Message, 'sending_client_id'), CountStat.DAY, False),
+    'messages_sent_to_stream:is_bot:hour': CountStat(
+        'messages_sent_to_stream:is_bot', zerver_count_message_by_stream, {},
+        (UserProfile, 'is_bot'), CountStat.HOUR, False)
+}
--- a/analytics/lib/fixtures.py
+++ b/analytics/lib/fixtures.py
@@ -1,14 +1,20 @@
+from __future__ import division, absolute_import
+
+from zerver.models import Realm, UserProfile, Stream, Message
+from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+
+from datetime import datetime
 from math import sqrt
 from random import gauss, random, seed
-from typing import List

-from analytics.lib.counts import CountStat
+from six.moves import range, zip

-def generate_time_series_data(days: int=100, business_hours_base: float=10,
-                              non_business_hours_base: float=10, growth: float=1,
-                              autocorrelation: float=0, spikiness: float=1,
-                              holiday_rate: float=0, frequency: str=CountStat.DAY,
-                              partial_sum: bool=False, random_seed: int=26) -> List[int]:
+def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
+                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
+                              frequency=CountStat.DAY, is_gauge=False, random_seed=26):
+    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
    """
    Generate semi-realistic looking time series data for testing analytics graphs.

@@ -25,7 +31,7 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
        the variance.
    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
    frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
+    is_gauge -- If True, return partial sum of the series.
    random_seed -- Seed for random number generator.
    """
    if frequency == CountStat.HOUR:
@@ -43,10 +49,10 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
                      [24*non_business_hours_base] * 2
        holidays = [random() < holiday_rate for i in range(days)]
    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
+        raise ValueError("Unknown frequency: %s" % (frequency,))
    if length < 2:
-        raise AssertionError("Must be generating at least 2 data points. "
-                             "Currently generating %s" % (length,))
+        raise ValueError("Must be generating at least 2 data points. "
+                         "Currently generating %s" % (length,))
    growth_base = growth ** (1. / (length-1))
    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]

@@ -57,7 +63,7 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,

    values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
              for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-    if partial_sum:
+    if is_gauge:
        for i in range(1, length):
            values[i] = values[i-1] + values[i]
    return [max(v, 0) for v in values]
--- a/analytics/lib/time_utils.py
+++ b/analytics/lib/time_utils.py
@@ -1,17 +1,15 @@
+from zerver.lib.timestamp import floor_to_hour, floor_to_day, timestamp_to_datetime
+from analytics.lib.counts import CountStat
+
 from datetime import datetime, timedelta
 from typing import List, Optional

-from analytics.lib.counts import CountStat
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
-
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(start: datetime, end: datetime, frequency: str,
-               min_length: Optional[int]) -> List[datetime]:
-    verify_UTC(start)
-    verify_UTC(end)
+def time_range(start, end, frequency, min_length):
+    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
    if frequency == CountStat.HOUR:
        end = floor_to_hour(end)
        step = timedelta(hours=1)
@@ -19,7 +17,7 @@ def time_range(start: datetime, end: datetime, frequency: str,
        end = floor_to_day(end)
        step = timedelta(days=1)
    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
+        raise ValueError("Unknown frequency: %s" % (frequency,))

    times = []
    if min_length is not None:
--- a/analytics/management/commands/active_user_stats.py
+++ b/analytics/management/commands/active_user_stats.py
@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from typing import Any
+
+from zerver.models import UserPresence, UserActivity
+from zerver.lib.utils import statsd, statsd_key
+
+from datetime import datetime, timedelta
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = """Sends active user statistics to statsd.
+
+    Run as a cron job that runs every 10 minutes."""
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        # Get list of all active users in the last 1 week
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+
+        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
+
+        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
+        hour_buckets = [0.16, 2, 12, 24, 48, 168]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+
+        for last_presence in users:
+            if last_presence.status == UserPresence.IDLE:
+                known_active = last_presence.timestamp - timedelta(minutes=30)
+            else:
+                known_active = last_presence.timestamp
+
+            for bucket in hour_buckets:
+                if bucket not in user_info[last_presence.user_profile.realm.string_id]:
+                    user_info[last_presence.user_profile.realm.string_id][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.string_id][bucket].append(last_presence.user_profile.email)
+
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
+
+        # Also do stats for how many users have been reading the app.
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
+        user_info = defaultdict(dict)
+        for activity in users_reading:
+            for bucket in hour_buckets:
+                if bucket not in user_info[activity.user_profile.realm.string_id]:
+                    user_info[activity.user_profile.realm.string_id][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.string_id][bucket].append(activity.user_profile.email)
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers reading for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
--- a/analytics/management/commands/active_user_stats_by_day.py
+++ b/analytics/management/commands/active_user_stats_by_day.py
@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import datetime
+import pytz
+
+from optparse import make_option
+from typing import Any
+from django.core.management.base import BaseCommand, CommandParser
+from zerver.lib.statistics import activity_averages_during_day
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity for a given day."
+
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
+        parser.add_argument('--date', default=None, action='store',
+                            help="Day to query in format 2013-12-05.  Default is yesterday")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options["date"] is None:
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")
--- a/analytics/management/commands/analyze_mit.py
+++ b/analytics/management/commands/analyze_mit.py
@@ -1,21 +1,25 @@
-import datetime
-import logging
-import time
-from typing import Any, Dict
+from __future__ import absolute_import
+from __future__ import print_function

+from typing import Any
+
+from optparse import make_option
 from django.core.management.base import BaseCommand, CommandParser
-
+from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Message, Recipient
+import datetime
+import time
+import logging

-def compute_stats(log_level: int) -> None:
+def compute_stats(log_level):
+    # type: (int) -> None
    logger = logging.getLogger()
    logger.setLevel(log_level)

    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
+    mit_query = Message.objects.filter(sender__realm__string_id="mit",
                                       recipient__type=Recipient.STREAM,
-                                       date_sent__gt=one_week_ago)
+                                       pub_date__gt=one_week_ago)
    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
        mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
    # Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
@@ -26,15 +30,15 @@ def compute_stats(log_level: int) -> None:
                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                           "root@mit.edu", "nagios@mit.edu",
                           "www-data|local-realm@mit.edu"])
-    user_counts = {}  # type: Dict[str, Dict[str, int]]
+    user_counts = {} # type: Dict[str, Dict[str, int]]
    for m in mit_query.select_related("sending_client", "sender"):
        email = m.sender.email
        user_counts.setdefault(email, {})
        user_counts[email].setdefault(m.sending_client.name, 0)
        user_counts[email][m.sending_client.name] += 1

-    total_counts = {}  # type: Dict[str, int]
-    total_user_counts = {}  # type: Dict[str, int]
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
    for email, counts in user_counts.items():
        total_user_counts.setdefault(email, 0)
        for client_name, count in counts.items():
@@ -43,7 +47,7 @@ def compute_stats(log_level: int) -> None:
            total_user_counts[email] += count

    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}  # type: Dict[int, float]
+    top_percents = {} # type: Dict[int, float]
    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
        top_percents[size] = 0.0
    for i, email in enumerate(sorted(total_user_counts.keys(),
@@ -71,10 +75,12 @@ def compute_stats(log_level: int) -> None:
 class Command(BaseCommand):
    help = "Compute statistics on MIT Zephyr usage."

-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
        parser.add_argument('--verbose', default=False, action='store_true')

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        level = logging.INFO
        if options["verbose"]:
            level = logging.DEBUG
--- a/analytics/management/commands/analyze_user_activity.py
+++ b/analytics/management/commands/analyze_user_activity.py
@@ -1,13 +1,19 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 from typing import Any, Dict

+from zerver.lib.statistics import seconds_usage_between
+
+from optparse import make_option
 from django.core.management.base import BaseCommand, CommandParser
+from zerver.models import UserProfile
+import datetime
 from django.utils.timezone import utc

-from zerver.lib.statistics import seconds_usage_between
-from zerver.models import UserProfile
-
-def analyze_activity(options: Dict[str, Any]) -> None:
+def analyze_activity(options):
+    # type: (Dict[str, Any]) -> None
    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
    day_end = day_start + datetime.timedelta(days=options["duration"])

@@ -46,11 +52,13 @@ Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""

-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
        parser.add_argument('--realm', action='store')
        parser.add_argument('--date', action='store', default="2013-09-06")
        parser.add_argument('--duration', action='store', default=1, type=int,
                            help="How many days to show usage information for")

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        analyze_activity(options)
--- a/analytics/management/commands/check_analytics_state.py
+++ b/analytics/management/commands/check_analytics_state.py
@@ -1,82 +0,0 @@
-from datetime import timedelta
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.models import installation_epoch, \
-    last_successful_fill
-from analytics.lib.counts import COUNT_STATS, CountStat
-from zerver.lib.timestamp import floor_to_hour, floor_to_day, verify_UTC, \
-    TimezoneNotUTCException
-from zerver.models import Realm
-
-import os
-import time
-from typing import Any, Dict
-
-states = {
-    0: "OK",
-    1: "WARNING",
-    2: "CRITICAL",
-    3: "UNKNOWN"
-}
-
-class Command(BaseCommand):
-    help = """Checks FillState table.
-
-    Run as a cron job that runs every hour."""
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        fill_state = self.get_fill_state()
-        status = fill_state['status']
-        message = fill_state['message']
-
-        state_file_path = "/var/lib/nagios_state/check-analytics-state"
-        state_file_tmp = state_file_path + "-tmp"
-
-        with open(state_file_tmp, "w") as f:
-            f.write("%s|%s|%s|%s\n" % (
-                int(time.time()), status, states[status], message))
-        os.rename(state_file_tmp, state_file_path)
-
-    def get_fill_state(self) -> Dict[str, Any]:
-        if not Realm.objects.exists():
-            return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
-
-        warning_unfilled_properties = []
-        critical_unfilled_properties = []
-        for property, stat in COUNT_STATS.items():
-            last_fill = last_successful_fill(property)
-            if last_fill is None:
-                last_fill = installation_epoch()
-            try:
-                verify_UTC(last_fill)
-            except TimezoneNotUTCException:
-                return {'status': 2, 'message': 'FillState not in UTC for %s' % (property,)}
-
-            if stat.frequency == CountStat.DAY:
-                floor_function = floor_to_day
-                warning_threshold = timedelta(hours=26)
-                critical_threshold = timedelta(hours=50)
-            else:  # CountStat.HOUR
-                floor_function = floor_to_hour
-                warning_threshold = timedelta(minutes=90)
-                critical_threshold = timedelta(minutes=150)
-
-            if floor_function(last_fill) != last_fill:
-                return {'status': 2, 'message': 'FillState not on %s boundary for %s' %
-                        (stat.frequency, property)}
-
-            time_to_last_fill = timezone_now() - last_fill
-            if time_to_last_fill > critical_threshold:
-                critical_unfilled_properties.append(property)
-            elif time_to_last_fill > warning_threshold:
-                warning_unfilled_properties.append(property)
-
-        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return {'status': 0, 'message': 'FillState looks fine.'}
-        if len(critical_unfilled_properties) == 0:
-            return {'status': 1, 'message': 'Missed filling %s once.' %
-                    (', '.join(warning_unfilled_properties),)}
-        return {'status': 2, 'message': 'Missed filling %s once. Missed filling %s at least twice.' %
-                (', '.join(warning_unfilled_properties), ', '.join(critical_unfilled_properties))}
--- a/analytics/management/commands/clear_analytics_tables.py
+++ b/analytics/management/commands/clear_analytics_tables.py
@@ -1,20 +1,29 @@
-from argparse import ArgumentParser
-from typing import Any
+from __future__ import absolute_import
+from __future__ import print_function

-from django.core.management.base import BaseCommand, CommandError
+import sys
+
+from argparse import ArgumentParser
+from django.db import connection
+from django.core.management.base import BaseCommand

 from analytics.lib.counts import do_drop_all_analytics_tables

+from typing import Any
+
 class Command(BaseCommand):
    help = """Clear analytics tables."""

-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
        parser.add_argument('--force',
                            action='store_true',
                            help="Clear analytics tables.")

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        if options['force']:
            do_drop_all_analytics_tables()
        else:
-            raise CommandError("Would delete all data from analytics tables (!); use --force to do so.")
+            print("Would delete all data from analytics tables (!); use --force to do so.")
+            sys.exit(1)
--- a/analytics/management/commands/clear_single_stat.py
+++ b/analytics/management/commands/clear_single_stat.py
@@ -1,26 +0,0 @@
-from argparse import ArgumentParser
-from typing import Any
-
-from django.core.management.base import BaseCommand, CommandError
-
-from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument('--force',
-                            action='store_true',
-                            help="Actually do it.")
-        parser.add_argument('--property',
-                            type=str,
-                            help="The property of the stat to be cleared.")
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        property = options['property']
-        if property not in COUNT_STATS:
-            raise CommandError("Invalid property: %s" % (property,))
-        if not options['force']:
-            raise CommandError("No action taken. Use --force.")
-
-        do_drop_single_stat(property)
--- a/analytics/management/commands/client_activity.py
+++ b/analytics/management/commands/client_activity.py
@@ -1,32 +1,33 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
 from argparse import ArgumentParser
-from typing import Any, Optional
-
+from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
-from django.utils.timezone import now as timezone_now

-from zerver.lib.management import ZulipBaseCommand
-from zerver.models import UserActivity
+from zerver.models import UserActivity, UserProfile, Realm, \
+    get_realm, get_user_profile_by_email

-class Command(ZulipBaseCommand):
+import datetime
+
+class Command(BaseCommand):
    help = """Report rough client activity globally, for a realm, or for a user

 Usage examples:

-./manage.py client_activity --target server
-./manage.py client_activity --target realm --realm zulip
-./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
+./manage.py client_activity
+./manage.py client_activity zulip
+./manage.py client_activity hamlet@zulip.com"""

-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument('--target', dest='target', required=True, type=str,
-                            help="'server' will calculate client activity of the entire server. "
-                                 "'realm' will calculate client activity of realm. "
-                                 "'user' will calculate client activity of the user.")
-        parser.add_argument('--user', dest='user', type=str,
-                            help="The email address of the user you want to calculate activity.")
-        self.add_realm_args(parser)
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
+                            help="realm or user to estimate client activity for")

-    def compute_activity(self, user_activity_objects: QuerySet) -> None:
+    def compute_activity(self, user_activity_objects):
+        # type: (QuerySet) -> None
        # Report data from the past week.
        #
        # This is a rough report of client activity because we inconsistently
@@ -37,7 +38,7 @@ Usage examples:
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
-        threshold = timezone_now() - datetime.timedelta(days=7)
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))
@@ -56,18 +57,24 @@ Usage examples:
            print("%25s %15d" % (count[1], count[0]))
        print("Total:", total)

-    def handle(self, *args: Any, **options: Optional[str]) -> None:
-        realm = self.get_realm(options)
-        if options["user"] is None:
-            if options["target"] == "server" and realm is None:
-                # Report global activity.
-                self.compute_activity(UserActivity.objects.all())
-            elif options["target"] == "realm" and realm is not None:
-                self.compute_activity(UserActivity.objects.filter(user_profile__realm=realm))
-            else:
-                self.print_help("./manage.py", "client_activity")
-        elif options["target"] == "user":
-            user_profile = self.get_user(options["user"], realm)
-            self.compute_activity(UserActivity.objects.filter(user_profile=user_profile))
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['arg'] is None:
+            # Report global activity.
+            self.compute_activity(UserActivity.objects.all())
        else:
-            self.print_help("./manage.py", "client_activity")
+            arg = options['arg']
+            try:
+                # Report activity for a user.
+                user_profile = get_user_profile_by_email(arg)
+                self.compute_activity(UserActivity.objects.filter(
+                    user_profile=user_profile))
+            except UserProfile.DoesNotExist:
+                try:
+                    # Report activity for a realm.
+                    realm = get_realm(arg)
+                    self.compute_activity(UserActivity.objects.filter(
+                        user_profile__realm=realm))
+                except Realm.DoesNotExist:
+                    print("Unknown user or realm %s" % (arg,))
+                    exit(1)
--- a/analytics/management/commands/populate_analytics_db.py
+++ b/analytics/management/commands/populate_analytics_db.py
@@ -1,21 +1,22 @@
-from datetime import timedelta
-from typing import Any, Dict, List, Mapping, Optional, Type
-import mock
+from __future__ import absolute_import, print_function
+
+from argparse import ArgumentParser

 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
+from django.utils import timezone

-from analytics.lib.counts import COUNT_STATS, \
-    CountStat, do_drop_all_analytics_tables
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount
+from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, FillState, RealmCount, UserCount, \
-    StreamCount, InstallationCount
-from zerver.lib.actions import do_change_is_admin, STREAM_ASSIGNMENT_COLORS
-from zerver.lib.create_user import create_user
 from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, Stream, Client, \
-    Recipient, Subscription
+from zerver.models import Realm, UserProfile, Stream, Message, Client
+
+from datetime import datetime, timedelta
+
+from six.moves import zip
+from typing import Any, List, Optional, Text, Type, Union

 class Command(BaseCommand):
    help = """Populates analytics tables with randomly generated data."""
@@ -23,194 +24,100 @@ class Command(BaseCommand):
    DAYS_OF_DATA = 100
    random_seed = 26

-    def generate_fixture_data(self, stat: CountStat, business_hours_base: float,
-                              non_business_hours_base: float, growth: float,
-                              autocorrelation: float, spikiness: float,
-                              holiday_rate: float=0, partial_sum: bool=False) -> List[int]:
+    def create_user(self, email, full_name, is_staff, date_joined, realm):
+        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
+        return UserProfile.objects.create(
+            email=email, full_name=full_name, is_staff=is_staff,
+            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
+            api_key='42', date_joined=date_joined)
+
+    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
+                              growth, autocorrelation, spikiness, holiday_rate=0):
+        # type: (CountStat, float, float, float, float, float, float) -> List[int]
        self.random_seed += 1
        return generate_time_series_data(
            days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
            non_business_hours_base=non_business_hours_base, growth=growth,
            autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
+            frequency=stat.frequency, is_gauge=(stat.interval == CountStat.GAUGE),
+            random_seed=self.random_seed)

-    def handle(self, *args: Any, **options: Any) -> None:
-        # TODO: This should arguably only delete the objects
-        # associated with the "analytics" realm.
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        do_drop_all_analytics_tables()
-
-        # This also deletes any objects with this realm as a foreign key
+        # I believe this also deletes any objects with this realm as a foreign key
        Realm.objects.filter(string_id='analytics').delete()
+        Client.objects.filter(name__endswith='_').delete()

-        # Because we just deleted a bunch of objects in the database
-        # directly (rather than deleting individual objects in Django,
-        # in which case our post_save hooks would have flushed the
-        # individual objects from memcached for us), we need to flush
-        # memcached in order to ensure deleted objects aren't still
-        # present in the memcached cache.
-        from zerver.apps import flush_cache
-        flush_cache(None)
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
+        installation_time = timezone.now() - timedelta(days=self.DAYS_OF_DATA)
+        last_end_time = floor_to_day(timezone.now())
        realm = Realm.objects.create(
-            string_id='analytics', name='Analytics', date_created=installation_time)
-        with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time):
-            shylock = create_user('shylock@analytics.ds', 'Shylock', realm,
-                                  full_name='Shylock', short_name='shylock',
-                                  is_realm_admin=True)
-        do_change_is_admin(shylock, True)
-        stream = Stream.objects.create(
-            name='all', realm=realm, date_created=installation_time)
-        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
-        stream.recipient = recipient
-        stream.save(update_fields=["recipient"])
+            string_id='analytics', name='Analytics', domain='analytics.ds',
+            date_created=installation_time)
+        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)

-        # Subscribe shylock to the stream to avoid invariant failures.
-        # TODO: This should use subscribe_users_to_streams from populate_db.
-        subs = [
-            Subscription(recipient=recipient,
-                         user_profile=shylock,
-                         color=STREAM_ASSIGNMENT_COLORS[0]),
-        ]
-        Subscription.objects.bulk_create(subs)
-
-        def insert_fixture_data(stat: CountStat,
-                                fixture_data: Mapping[Optional[str], List[int]],
-                                table: Type[BaseCount]) -> None:
+        def insert_fixture_data(stat, fixture_data, table):
+            # type: (CountStat, Dict[Optional[str], List[int]], Type[BaseCount]) -> None
            end_times = time_range(last_end_time, last_end_time, stat.frequency,
                                   len(list(fixture_data.values())[0]))
-            if table == InstallationCount:
-                id_args = {}  # type: Dict[str, Any]
            if table == RealmCount:
                id_args = {'realm': realm}
            if table == UserCount:
                id_args = {'realm': realm, 'user': shylock}
-            if table == StreamCount:
-                id_args = {'stream': stream, 'realm': realm}
-
            for subgroup, values in fixture_data.items():
                table.objects.bulk_create([
                    table(property=stat.property, subgroup=subgroup, end_time=end_time,
                          value=value, **id_args)
                    for end_time, value in zip(end_times, values) if value != 0])

-        stat = COUNT_STATS['1day_actives::day']
+        stat = COUNT_STATS['active_users:is_bot:day']
        realm_data = {
-            None: self.generate_fixture_data(stat, .08, .02, 3, .3, 6, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
+            'false': self.generate_fixture_data(stat, .1, .03, 3, .5, 3),
+            'true': self.generate_fixture_data(stat, .01, 0, 1, 0, 1)
+        } # type: Dict[Optional[str], List[int]]
        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, .8, .2, 4, .3, 6, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['realm_active_humans::day']
-        realm_data = {
-            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 1, .3, 4, .5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        realm_data = {
-            'false': self.generate_fixture_data(stat, .1, .03, 3.5, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'false': self.generate_fixture_data(stat, 1, .3, 6, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)

        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data = {'false': self.generate_fixture_data(
-            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
+        user_data = {'false': self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
                      'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {'false': self.generate_fixture_data(stat, 350, 150, 6, .6, 4),
-                             'true': self.generate_fixture_data(stat, 150, 150, 3, .4, 2)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)

        stat = COUNT_STATS['messages_sent:message_type:day']
        user_data = {
            'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
-            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
+            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8)}
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
            'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
+            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4)}
        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'public_stream': self.generate_fixture_data(stat, 300, 80, 5, .6, 4),
-            'private_stream': self.generate_fixture_data(stat, 70, 70, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 130, 50, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, .6, 4)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)

-        website, created = Client.objects.get_or_create(name='website')
-        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
-        android, created = Client.objects.get_or_create(name='ZulipAndroid')
-        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
-        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
-        API, created = Client.objects.get_or_create(name='API: Python')
-        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
-        unused, created = Client.objects.get_or_create(name='unused')
-        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
+        website_ = Client.objects.create(name='website_')
+        API_ = Client.objects.create(name='API_')
+        android_ = Client.objects.create(name='android_')
+        iOS_ = Client.objects.create(name='iOS_')
+        react_native_ = Client.objects.create(name='react_native_')
+        electron_ = Client.objects.create(name='electron_')
+        barnowl_ = Client.objects.create(name='barnowl_')
+        plan9_ = Client.objects.create(name='plan9_')

        stat = COUNT_STATS['messages_sent:client:day']
        user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
+            website_.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
+            barnowl_.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
+            website_.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
+            API_.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
+            android_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            iOS_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            react_native_.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
+            electron_.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
+            barnowl_.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
+            plan9_.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0, 0)}
        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            website.id: self.generate_fixture_data(stat, 300, 200, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 50, 50, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)

-        stat = COUNT_STATS['messages_in_stream:is_bot:day']
-        realm_data = {'false': self.generate_fixture_data(stat, 30, 5, 6, .6, 4),
-                      'true': self.generate_fixture_data(stat, 20, 2, 3, .2, 3)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data = {'false': self.generate_fixture_data(stat, 10, 7, 5, .6, 4),
-                       'true': self.generate_fixture_data(stat, 5, 3, 2, .4, 2)}  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, stream_data, StreamCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
+        # TODO: messages_sent_to_stream:is_bot
--- a/analytics/management/commands/realm_stats.py
+++ b/analytics/management/commands/realm_stats.py
@@ -1,13 +1,17 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any
+
 from argparse import ArgumentParser
-from typing import Any, List
+import datetime
+import pytz

-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 from django.db.models import Count
-from django.utils.timezone import now as timezone_now
-
-from zerver.models import Message, Realm, Recipient, Stream, \
-    Subscription, UserActivity, UserMessage, UserProfile, get_realm
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
+    Subscription, UserMessage, get_realm

 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
@@ -17,13 +21,15 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
 class Command(BaseCommand):
    help = "Generate statistics on realm activity."

-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")

-    def active_users(self, realm: Realm) -> List[UserProfile]:
+    def active_users(self, realm):
+        # type: (Realm) -> List[UserProfile]
        # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
        return [activity.user_profile for activity in (
            UserActivity.objects.filter(user_profile__realm=realm,
                                        user_profile__is_active=True,
@@ -31,49 +37,59 @@ class Command(BaseCommand):
                                        query="/json/users/me/pointer",
                                        client__name="website"))]

-    def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender=user, date_sent__gt=sent_time_cutoff).count()
+    def messages_sent_by(self, user, days_ago):
+        # type: (UserProfile, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()

-    def total_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return Message.objects.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
+    def total_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

-    def human_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
+    def human_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

-    def api_messages(self, realm: Realm, days_ago: int) -> int:
+    def api_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
        return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))

-    def stream_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff,
+    def stream_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                     recipient__type=Recipient.STREAM).count()

-    def private_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
+    def private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()

-    def group_private_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
+    def group_private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()

-    def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
+    def report_percentage(self, numerator, denominator, text):
+        # type: (float, float, str) -> None
        if not denominator:
            fraction = 0.0
        else:
            fraction = numerator / float(denominator)
        print("%.2f%% of" % (fraction * 100,), text)

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
                realms = [get_realm(string_id) for string_id in options['realms']]
            except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
        else:
            realms = Realm.objects.all()

@@ -130,7 +146,7 @@ class Command(BaseCommand):
                user_profile__in=user_profiles, active=True)

            # Streams not in home view
-            non_home_view = active_user_subs.filter(is_muted=True).values(
+            non_home_view = active_user_subs.filter(in_home_view=False).values(
                "user_profile").annotate(count=Count("user_profile"))
            print("%d users have %d streams not in home view" % (
                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
@@ -143,7 +159,7 @@ class Command(BaseCommand):
                len(markup_messages), sum([elt["count"] for elt in markup_messages])))

            # Notifications for stream messages
-            notifications = active_user_subs.filter(desktop_notifications=True).values(
+            notifications = active_user_subs.filter(notifications=True).values(
                "user_profile").annotate(count=Count("user_profile"))
            print("%d users receive desktop notifications for %d streams" % (
                len(notifications), sum([elt["count"] for elt in notifications])))
--- a/analytics/management/commands/stream_stats.py
+++ b/analytics/management/commands/stream_stats.py
@@ -1,56 +1,46 @@
-from argparse import ArgumentParser
+from __future__ import absolute_import
+from __future__ import print_function
+
 from typing import Any

-from django.core.management.base import BaseCommand, CommandError
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
 from django.db.models import Q
-
-from zerver.models import Message, Realm, \
-    Recipient, Stream, Subscription, get_realm
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm

 class Command(BaseCommand):
    help = "Generate statistics on the streams for a realm."

-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")

-    def handle(self, *args: Any, **options: str) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
        if options['realms']:
            try:
                realms = [get_realm(string_id) for string_id in options['realms']]
            except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
        else:
            realms = Realm.objects.all()

        for realm in realms:
+            print(realm.string_id)
+            print("------------")
+            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
-            # private stream count
-            private_count = 0
-            # public stream count
-            public_count = 0
+            invite_only_count = 0
            for stream in streams:
                if stream.invite_only:
-                    private_count += 1
-                else:
-                    public_count += 1
-            print("------------")
-            print(realm.string_id, end=' ')
-            print("%10s %d public streams and" % ("(", public_count), end=' ')
-            print("%d private streams )" % (private_count,))
-            print("------------")
-            print("%25s %15s %10s %12s" % ("stream", "subscribers", "messages", "type"))
-
-            for stream in streams:
-                if stream.invite_only:
-                    stream_type = 'private'
-                else:
-                    stream_type = 'public'
+                    invite_only_count += 1
+                    continue
                print("%25s" % (stream.name,), end=' ')
                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
-                print("%10d" % (len(Subscription.objects.filter(recipient=recipient,
-                                                                active=True)),), end=' ')
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
                num_messages = len(Message.objects.filter(recipient=recipient))
-                print("%12d" % (num_messages,), end=' ')
-                print("%15s" % (stream_type,))
+                print("%12d" % (num_messages,))
+            print("%d invite-only streams" % (invite_only_count,))
            print("")
--- a/analytics/management/commands/update_analytics_counts.py
+++ b/analytics/management/commands/update_analytics_counts.py
@@ -1,44 +1,49 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 import os
-import time
-from argparse import ArgumentParser
-from typing import Any, Dict
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from django.utils.dateparse import parse_datetime
-from django.utils.timezone import now as timezone_now
-from django.utils.timezone import utc as timezone_utc
-
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+import sys
 from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.remote_server import send_analytics_to_remote_server
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import Realm
+
+from argparse import ArgumentParser
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+from django.conf import settings
+
+from analytics.models import RealmCount, UserCount
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from zerver.lib.timestamp import datetime_to_string, is_timezone_aware
+from zerver.models import UserProfile, Message
+
+from typing import Any

 class Command(BaseCommand):
    help = """Fills Analytics tables.

    Run as a cron job that runs every hour."""

-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
        parser.add_argument('--time', '-t',
                            type=str,
-                            help='Update stat tables from current state to'
-                                 '--time. Defaults to the current time.',
-                            default=timezone_now().isoformat())
+                            help='Update stat tables from current state to --time. Defaults to the current time.',
+                            default=datetime_to_string(timezone.now()))
        parser.add_argument('--utc',
-                            action='store_true',
+                            type=bool,
                            help="Interpret --time in UTC.",
                            default=False)
        parser.add_argument('--stat', '-s',
                            type=str,
                            help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument('--verbose',
-                            action='store_true',
-                            help="Print timing information to stdout.",
-                            default=False)
+        parser.add_argument('--quiet', '-q',
+                            type=str,
+                            help="Suppress output to stdout.")

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        try:
            os.mkdir(settings.ANALYTICS_LOCK_DIR)
        except OSError:
@@ -50,41 +55,21 @@ class Command(BaseCommand):
        finally:
            os.rmdir(settings.ANALYTICS_LOCK_DIR)

-    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
-        # installation_epoch relies on there being at least one realm; we
-        # shouldn't run the analytics code if that condition isn't satisfied
-        if not Realm.objects.exists():
-            logger.info("No realms, stopping update_analytics_counts")
-            return
-
+    def run_update_analytics_counts(self, options):
+        # type: (Dict[str, Any]) -> None
        fill_to_time = parse_datetime(options['time'])
        if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
-        if fill_to_time.tzinfo is None:
+            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
+
+        if not (is_timezone_aware(fill_to_time)):
            raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")

-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
+        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))

        if options['stat'] is not None:
-            stats = [COUNT_STATS[options['stat']]]
+            process_count_stat(COUNT_STATS[options['stat']], fill_to_time)
        else:
-            stats = list(COUNT_STATS.values())
+            for stat in COUNT_STATS.values():
+                process_count_stat(stat, fill_to_time)

-        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
-        if options['verbose']:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options['verbose']:
-                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
-                last = time.time()
-
-        if options['verbose']:
-            print("Finished updating analytics counts through %s in %.3fs" %
-                  (fill_to_time, time.time() - start))
        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
-
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
-            send_analytics_to_remote_server()
--- a/analytics/management/commands/user_stats.py
+++ b/analytics/management/commands/user_stats.py
@@ -1,30 +1,37 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import print_function
+
 from argparse import ArgumentParser
+import datetime
+import pytz
 from typing import Any

-from django.core.management.base import BaseCommand, CommandError
-from django.utils.timezone import now as timezone_now
-
-from zerver.models import Message, Realm, Stream, UserProfile, get_realm
+from django.core.management.base import BaseCommand
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range

 class Command(BaseCommand):
    help = "Generate statistics on user activity."

-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")

-    def messages_sent_by(self, user: UserProfile, week: int) -> int:
-        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
-        end = timezone_now() - datetime.timedelta(days=week*7)
-        return Message.objects.filter(sender=user, date_sent__gt=start, date_sent__lte=end).count()
+    def messages_sent_by(self, user, week):
+        # type: (UserProfile, int) -> int
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
+        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()

-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
                realms = [get_realm(string_id) for string_id in options['realms']]
            except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
        else:
            realms = Realm.objects.all()

@@ -37,5 +44,5 @@ class Command(BaseCommand):
            for user_profile in user_profiles:
                print("%35s" % (user_profile.email,), end=' ')
                for week in range(10):
-                    print("%5d" % (self.messages_sent_by(user_profile, week),), end=' ')
+                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
                print("")
--- a/analytics/migrations/0001_initial.py
+++ b/analytics/migrations/0001_initial.py
@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
-import django.db.models.deletion
+from __future__ import unicode_literals
+
+from django.db import models, migrations
 from django.conf import settings
-from django.db import migrations, models
+import zerver.lib.str_utils
+

 class Migration(migrations.Migration):

@@ -17,21 +20,21 @@ class Migration(migrations.Migration):
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('info', models.CharField(max_length=1000)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.CreateModel(
            name='HuddleCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('huddle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('huddle', models.ForeignKey(to='zerver.Recipient')),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.CreateModel(
            name='InstallationCount',
@@ -41,51 +44,51 @@ class Migration(migrations.Migration):
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.CreateModel(
            name='RealmCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),

            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.CreateModel(
            name='StreamCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream')),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
+                ('stream', models.ForeignKey(to='zerver.Stream')),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.CreateModel(
            name='UserCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
        migrations.AlterUniqueTogether(
            name='usercount',
--- a/analytics/migrations/0002_remove_huddlecount.py
+++ b/analytics/migrations/0002_remove_huddlecount.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
-from django.db import migrations
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+

 class Migration(migrations.Migration):

--- a/analytics/migrations/0003_fillstate.py
+++ b/analytics/migrations/0003_fillstate.py
@@ -1,5 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
+import zerver.lib.str_utils
+

 class Migration(migrations.Migration):

@@ -17,6 +21,6 @@ class Migration(migrations.Migration):
                ('state', models.PositiveSmallIntegerField()),
                ('last_modified', models.DateTimeField(auto_now=True)),
            ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
        ),
    ]
--- a/analytics/migrations/0004_add_subgroup.py
+++ b/analytics/migrations/0004_add_subgroup.py
@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models

+
 class Migration(migrations.Migration):

    dependencies = [
--- a/analytics/migrations/0005_alter_field_size.py
+++ b/analytics/migrations/0005_alter_field_size.py
@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models

+
 class Migration(migrations.Migration):

    dependencies = [
--- a/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
+++ b/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
-from django.db import migrations
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+

 class Migration(migrations.Migration):

--- a/analytics/migrations/0007_remove_interval.py
+++ b/analytics/migrations/0007_remove_interval.py
@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-16 20:50
+from __future__ import unicode_literals
+
+from django.conf import settings
 from django.db import migrations

+
 class Migration(migrations.Migration):

    dependencies = [
--- a/analytics/migrations/0008_add_count_indexes.py
+++ b/analytics/migrations/0008_add_count_indexes.py
@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-02-01 22:28
+from __future__ import unicode_literals
+
 from django.db import migrations

+
 class Migration(migrations.Migration):

    dependencies = [
--- a/analytics/migrations/0009_remove_messages_to_stream_stat.py
+++ b/analytics/migrations/0009_remove_messages_to_stream_stat.py
@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent_to_stream:is_bot'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0008_add_count_indexes'),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]
--- a/analytics/migrations/0010_clear_messages_sent_values.py
+++ b/analytics/migrations/0010_clear_messages_sent_values.py
@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent:message_type:day'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]
--- a/analytics/migrations/0011_clear_analytics_tables.py
+++ b/analytics/migrations/0011_clear_analytics_tables.py
@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0010_clear_messages_sent_values'),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]
--- a/analytics/migrations/0012_add_on_delete.py
+++ b/analytics/migrations/0012_add_on_delete.py
@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.6 on 2018-01-29 08:14
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0011_clear_analytics_tables'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='installationcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-    ]
--- a/analytics/migrations/0013_remove_anomaly.py
+++ b/analytics/migrations/0013_remove_anomaly.py
@@ -1,34 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.18 on 2019-02-02 02:47
-from __future__ import unicode_literals
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0012_add_on_delete'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='installationcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='realmcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='streamcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='usercount',
-            name='anomaly',
-        ),
-        migrations.DeleteModel(
-            name='Anomaly',
-        ),
-    ]
--- a/analytics/models.py
+++ b/analytics/models.py
@@ -1,92 +1,151 @@
-import datetime
-from typing import Optional
-
 from django.db import models
+from django.utils import timezone

-from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, Stream, UserProfile
+from zerver.models import Realm, UserProfile, Stream, Recipient
+from zerver.lib.str_utils import ModelReprMixin
+from zerver.lib.timestamp import datetime_to_UTC, floor_to_day

-class FillState(models.Model):
-    property = models.CharField(max_length=40, unique=True)  # type: str
-    end_time = models.DateTimeField()  # type: datetime.datetime
+import datetime
+
+from typing import Optional, Tuple, Union, Dict, Any, Text
+
+class FillState(ModelReprMixin, models.Model):
+    property = models.CharField(max_length=40, unique=True) # type: Text
+    end_time = models.DateTimeField() # type: datetime.datetime

    # Valid states are {DONE, STARTED}
    DONE = 1
    STARTED = 2
-    state = models.PositiveSmallIntegerField()  # type: int
+    state = models.PositiveSmallIntegerField() # type: int

-    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
+    last_modified = models.DateTimeField(auto_now=True) # type: datetime.datetime

-    def __str__(self) -> str:
-        return "<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)

 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime.datetime:
+def installation_epoch():
+    # type: () -> datetime.datetime
    earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    return floor_to_day(earliest_realm_creation)
+    return floor_to_day(datetime_to_UTC(earliest_realm_creation))

-def last_successful_fill(property: str) -> Optional[datetime.datetime]:
-    fillstate = FillState.objects.filter(property=property).first()
-    if fillstate is None:
-        return None
-    if fillstate.state == FillState.DONE:
-        return fillstate.end_time
-    return fillstate.end_time - datetime.timedelta(hours=1)
+# would only ever make entries here by hand
+class Anomaly(ModelReprMixin, models.Model):
+    info = models.CharField(max_length=1000) # type: Text

-class BaseCount(models.Model):
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<Anomaly: %s... %s>" % (self.info, self.id)
+
+class BaseCount(ModelReprMixin, models.Model):
    # Note: When inheriting from BaseCount, you may want to rearrange
    # the order of the columns in the migration to make sure they
    # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)  # type: str
-    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[str]
-    end_time = models.DateTimeField()  # type: datetime.datetime
-    value = models.BigIntegerField()  # type: int
+    property = models.CharField(max_length=32) # type: Text
+    subgroup = models.CharField(max_length=16, null=True) # type: Text
+    end_time = models.DateTimeField() # type: datetime.datetime
+    value = models.BigIntegerField() # type: int
+    anomaly = models.ForeignKey(Anomaly, null=True) # type: Optional[Anomaly]

-    class Meta:
+    class Meta(object):
        abstract = True

+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        raise NotImplementedError
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        raise NotImplementedError
+
 class InstallationCount(BaseCount):

-    class Meta:
+    class Meta(object):
        unique_together = ("property", "subgroup", "end_time")

-    def __str__(self) -> str:
-        return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ()
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return None
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)

 class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)

-    class Meta:
+    class Meta(object):
        unique_together = ("realm", "property", "subgroup", "end_time")
        index_together = ["property", "end_time"]

-    def __str__(self) -> str:
-        return "<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('realm_id',)
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Realm
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)

 class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    user = models.ForeignKey(UserProfile)
+    realm = models.ForeignKey(Realm)

-    class Meta:
+    class Meta(object):
        unique_together = ("user", "property", "subgroup", "end_time")
        # This index dramatically improves the performance of
        # aggregating from users to realms
        index_together = ["property", "realm", "end_time"]

-    def __str__(self) -> str:
-        return "<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('user_id', 'realm_id')
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return UserProfile
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)

 class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    stream = models.ForeignKey(Stream)
+    realm = models.ForeignKey(Realm)

-    class Meta:
+    class Meta(object):
        unique_together = ("stream", "property", "subgroup", "end_time")
        # This index dramatically improves the performance of
        # aggregating from streams to realms
        index_together = ["property", "realm", "end_time"]

-    def __str__(self) -> str:
-        return "<StreamCount: %s %s %s %s %s>" % (
-            self.stream, self.property, self.subgroup, self.value, self.id)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('stream_id', 'realm_id')
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Stream
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)
--- a/analytics/tests/test_counts.py
+++ b/analytics/tests/test_counts.py
--- a/analytics/tests/test_fixtures.py
+++ b/analytics/tests/test_fixtures.py
@@ -1,29 +0,0 @@
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-from zerver.lib.test_classes import ZulipTestCase
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self) -> None:
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=1500,
-            growth=2, spikiness=0, frequency=CountStat.HOUR)
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assertEqual(len(data), 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500*2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=2000,
-            autocorrelation=1, frequency=CountStat.HOUR)
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])
--- a/analytics/tests/test_views.py
+++ b/analytics/tests/test_views.py
@@ -1,578 +1,23 @@
-from datetime import datetime, timedelta
-from typing import List, Optional
-
-import mock
-from django.utils.timezone import utc
-from django.http import HttpResponse
-import ujson
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, \
-    RealmCount, UserCount, last_successful_fill
-from analytics.views import rewrite_client_arrays, \
-    sort_by_totals, sort_client_labels
+from django.utils.timezone import get_fixed_timezone
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_day, \
-    ceiling_to_hour, datetime_to_timestamp
-from zerver.lib.actions import do_create_multiuse_invite_link, \
-    do_send_realm_reactivation_email
-from zerver.models import Client, get_realm, MultiuseInvite

-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        result = self.client_get('/stats')
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+from analytics.views import rewrite_client_arrays

-    def test_guest_user_cant_access_stats(self) -> None:
-        self.user = self.example_user('polonius')
-        self.login(self.user.email)
-        result = self.client_get('/stats')
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-        result = self.client_get('/json/analytics/chart_data')
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-    def test_stats_for_realm(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 302)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/realm/not_existing_realm/')
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 302)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-class TestGetChartData(ZulipTestCase):
-    def setUp(self) -> None:
-        super().setUp()
-        self.realm = get_realm('zulip')
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
-                               for i in range(4)]
-        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
-                              for i in range(4)]
-
-    def data(self, i: int) -> List[int]:
-        return [0, 0, i, 0]
-
-    def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]],
-                    user_subgroups: List[str]) -> None:
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create([
-            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                       value=100+i, realm=self.realm)
-            for i, subgroup in enumerate(realm_subgroups)])
-        UserCount.objects.bulk_create([
-            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                      value=200+i, realm=self.realm, user=self.user)
-            for i, subgroup in enumerate(user_subgroups)])
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_over_time(self) -> None:
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        self.insert_data(stat, ['true', 'false'], ['false'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-            'frequency': CountStat.HOUR,
-            'everyone': {'bot': self.data(100), 'human': self.data(101)},
-            'user': {'bot': self.data(0), 'human': self.data(200)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_message_type(self) -> None:
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        self.insert_data(stat, ['public_stream', 'private_message'],
-                         ['public_stream', 'private_stream'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'Public streams': self.data(100), 'Private streams': self.data(0),
-                         'Private messages': self.data(101), 'Group private messages': self.data(0)},
-            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
-                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
-            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_client(self) -> None:
-        stat = COUNT_STATS['messages_sent:client:day']
-        client1 = Client.objects.create(name='client 1')
-        client2 = Client.objects.create(name='client 2')
-        client3 = Client.objects.create(name='client 3')
-        client4 = Client.objects.create(name='client 4')
-        self.insert_data(stat, [client4.id, client3.id, client2.id],
-                         [client3.id, client1.id])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'client 4': self.data(100), 'client 3': self.data(101),
-                         'client 2': self.data(102)},
-            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
-            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
-            'result': 'success',
-        })
-
-    def test_include_empty_subgroups(self) -> None:
-        FillState.objects.create(
-            property='realm_active_humans::day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {"_1day": [0], "_15day": [0], "all_time": [0]})
-        self.assertFalse('user' in data)
-
-        FillState.objects.create(
-            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {'human': [0], 'bot': [0]})
-        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:message_type:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
-        self.assertEqual(data['user'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:client:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {})
-        self.assertEqual(data['user'], {})
-
-    def test_start_and_end(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[1],
-                                  'end': end_time_timestamps[2]})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
-        self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})
-
-        # start later then end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[2],
-                                  'end': end_time_timestamps[1]})
-        self.assert_json_error_contains(result, 'Start time is later than')
-
-    def test_min_length(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        # test min_length is too short to change anything
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 2})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
-        self.assertEqual(data['everyone'], {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)})
-        # test min_length larger than filled data
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 5})
-        self.assert_json_success(result)
-        data = result.json()
-        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(data['everyone'], {'_1day': [0]+self.data(100), '_15day': [0]+self.data(100), 'all_time': [0]+self.data(100)})
-
-    def test_non_existent_chart(self) -> None:
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'does_not_exist'})
-        self.assert_json_error_contains(result, 'Unknown chart name')
-
-    def test_analytics_not_running(self) -> None:
-        # try to get data for a valid chart, but before we've put anything in the database
-        # (e.g. before update_analytics_counts has been run)
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'number_of_humans'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip/',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, 'Invalid organization', 400)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-class TestSupportEndpoint(ZulipTestCase):
-    def test_search(self) -> None:
-        def check_hamlet_user_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>',
-                                             '<b>Email</b>: hamlet@zulip.com', '<b>Is active</b>: True<br>',
-                                             '<b>Admins</b>: iago@zulip.com\n',
-                                             'class="copy-button" data-copytext="iago@zulip.com"'
-                                             ], result)
-
-        def check_zulip_realm_query_result(result: HttpResponse) -> None:
-            zulip_realm = get_realm("zulip")
-            self.assert_in_success_response(['<input type="hidden" name="realm_id" value="%s"' % (zulip_realm.id,),
-                                             'Zulip Dev</h3>',
-                                             '<option value="1" selected>Self Hosted</option>',
-                                             '<option value="2" >Limited</option>',
-                                             'input type="number" name="discount" value="None"',
-                                             '<option value="active" selected>Active</option>',
-                                             '<option value="deactivated" >Deactivated</option>',
-                                             'scrub-realm-button">',
-                                             'data-string-id="zulip"'], result)
-
-        def check_lear_realm_query_result(result: HttpResponse) -> None:
-            lear_realm = get_realm("lear")
-            self.assert_in_success_response(['<input type="hidden" name="realm_id" value="%s"' % (lear_realm.id,),
-                                             'Lear &amp; Co.</h3>',
-                                             '<option value="1" selected>Self Hosted</option>',
-                                             '<option value="2" >Limited</option>',
-                                             'input type="number" name="discount" value="None"',
-                                             '<option value="active" selected>Active</option>',
-                                             '<option value="deactivated" >Deactivated</option>',
-                                             'scrub-realm-button">',
-                                             'data-string-id="lear"'], result)
-
-        def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: Optional[bool]=False) -> None:
-            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
-                                             '<b>Email</b>: {}'.format(email),
-                                             ], result)
-            if invite:
-                self.assert_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3',
-                                                 '<b>Status</b>: Link has never been clicked'], result)
-                self.assert_in_success_response([], result)
-            else:
-                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day',
-                                                 '<b>Status</b>: Link has never been clicked'], result)
-
-        def check_realm_creation_query_result(result: HttpResponse, email: str) -> None:
-            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
-                                             '<span class="label">realm creation</span>\n',
-                                             '<b>Link</b>: http://zulip.testserver/accounts/do_confirm/',
-                                             '<b>Expires in</b>: 1\xa0day<br>\n'
-                                             ], result)
-
-        def check_multiuse_invite_link_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">multiuse invite</span>\n',
-                                             '<b>Link</b>: http://zulip.testserver/join/',
-                                             '<b>Expires in</b>: 1\xa0week, 3'
-                                             ], result)
-
-        def check_realm_reactivation_link_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">realm reactivation</span>\n',
-                                             '<b>Link</b>: http://zulip.testserver/reactivate/',
-                                             '<b>Expires in</b>: 1\xa0day'
-                                             ], result)
-
-        cordelia_email = self.example_email("cordelia")
-        self.login(cordelia_email)
-
-        result = self.client_get("/activity/support")
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago_email = self.example_email("iago")
-        self.login(iago_email)
-
-        result = self.client_get("/activity/support")
-        self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result)
-
-        result = self.client_get("/activity/support", {"q": "hamlet@zulip.com"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "lear"})
-        check_lear_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "http://lear.testserver"})
-        check_lear_realm_query_result(result)
-
-        with self.settings(REALM_HOSTS={'zulip': 'localhost'}):
-            result = self.client_get("/activity/support", {"q": "http://localhost"})
-            check_zulip_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "hamlet@zulip.com, lear"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "lear, Hamlet <hamlet@zulip.com>"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        self.client_post('/accounts/home/', {'email': self.nonreg_email("test")})
-        self.login(iago_email)
-        result = self.client_get("/activity/support", {"q": self.nonreg_email("test")})
-        check_preregistration_user_query_result(result, self.nonreg_email("test"))
-        check_zulip_realm_query_result(result)
-
-        stream_ids = [self.get_stream_id("Denmark")]
-        invitee_emails = [self.nonreg_email("test1")]
-        self.client_post("/json/invites", {"invitee_emails": invitee_emails,
-                         "stream_ids": ujson.dumps(stream_ids), "invite_as": 1})
-        result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")})
-        check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
-        check_zulip_realm_query_result(result)
-
-        email = self.nonreg_email('alice')
-        self.client_post('/new/', {'email': email})
-        result = self.client_get("/activity/support", {"q": email})
-        check_realm_creation_query_result(result, email)
-
-        do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1)
-        result = self.client_get("/activity/support", {"q": "zulip"})
-        check_multiuse_invite_link_query_result(result)
-        check_zulip_realm_query_result(result)
-        MultiuseInvite.objects.all().delete()
-
-        do_send_realm_reactivation_email(get_realm("zulip"))
-        result = self.client_get("/activity/support", {"q": "zulip"})
-        check_realm_reactivation_link_query_result(result)
-        check_zulip_realm_query_result(result)
-
-    def test_change_plan_type(self) -> None:
-        cordelia = self.example_user("cordelia")
-        self.login(cordelia.email)
-
-        result = self.client_post("/activity/support", {"realm_id": "%s" % (cordelia.realm_id,), "plan_type": "2"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login(iago.email)
-
-        with mock.patch("analytics.views.do_change_plan_type") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (iago.realm_id,), "plan_type": "2"})
-            m.assert_called_once_with(get_realm("zulip"), 2)
-            self.assert_in_success_response(["Plan type of Zulip Dev changed from self hosted to limited"], result)
-
-    def test_attach_discount(self) -> None:
-        lear_realm = get_realm("lear")
-        cordelia_email = self.example_email("cordelia")
-        self.login(cordelia_email)
-
-        result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago_email = self.example_email("iago")
-        self.login(iago_email)
-
-        with mock.patch("analytics.views.attach_discount_to_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
-            m.assert_called_once_with(get_realm("lear"), 25)
-            self.assert_in_success_response(["Discount of Lear &amp; Co. changed to 25 from None"], result)
-
-    def test_activate_or_deactivate_realm(self) -> None:
-        lear_realm = get_realm("lear")
-        cordelia_email = self.example_email("cordelia")
-        self.login(cordelia_email)
-
-        result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "deactivated"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago_email = self.example_email("iago")
-        self.login(iago_email)
-
-        with mock.patch("analytics.views.do_deactivate_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "deactivated"})
-            m.assert_called_once_with(lear_realm, self.example_user("iago"))
-            self.assert_in_success_response(["Lear &amp; Co. deactivated"], result)
-
-        with mock.patch("analytics.views.do_send_realm_reactivation_email") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "active"})
-            m.assert_called_once_with(lear_realm)
-            self.assert_in_success_response(["Realm reactivation email sent to admins of Lear"], result)
-
-    def test_scrub_realm(self) -> None:
-        lear_realm = get_realm("lear")
-        cordelia_email = self.example_email("cordelia")
-        self.login(cordelia_email)
-
-        result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago_email = self.example_email("iago")
-        self.login(iago_email)
-
-        with mock.patch("analytics.views.do_scrub_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "scrub_realm": "scrub_realm"})
-            m.assert_called_once_with(lear_realm)
-            self.assert_in_success_response(["Lear &amp; Co. scrubbed"], result)
-
-        with mock.patch("analytics.views.do_scrub_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,)})
-            m.assert_not_called()
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    # last_successful_fill is in analytics/models.py, but get_chart_data is
-    # the only function that uses it at the moment
-    def test_last_successful_fill(self) -> None:
-        self.assertIsNone(last_successful_fill('non-existant'))
-        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
-        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
-        fillstate = FillState.objects.create(property='property', end_time=a_time,
-                                             state=FillState.DONE)
-        self.assertEqual(last_successful_fill('property'), a_time)
-        fillstate.state = FillState.STARTED
-        fillstate.save()
-        self.assertEqual(last_successful_fill('property'), one_hour_before)
-
-    def test_sort_by_totals(self) -> None:
-        empty = []  # type: List[int]
-        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
-        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
-
-    def test_sort_client_labels(self) -> None:
-        data = {'everyone': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
-                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
-        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
+from datetime import datetime, timedelta

 class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
+    def test_time_range(self):
+        # type: () -> None
        HOUR = timedelta(hours=1)
        DAY = timedelta(days=1)
+        TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC

-        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
-        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
-        floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
+        # Using 22:59 so that converting to UTC and applying floor_to_{hour,day} do not commute
+        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)
+        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=TZINFO)
+        floor_day = datetime(2016, 3, 14).replace(tzinfo=TZINFO)

        # test start == end
        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
@@ -592,13 +37,13 @@ class TestTimeRange(ZulipTestCase):
                         [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])

 class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
+    def test_map_arrays(self):
+        # type: () -> None
        a = {'desktop app 1.0': [1, 2, 3],
             'desktop app 2.0': [10, 12, 13],
             'desktop app 3.0': [21, 22, 23],
             'website': [1, 2, 3],
             'ZulipiOS': [1, 2, 3],
-             'ZulipElectron': [2, 5, 7],
             'ZulipMobile': [1, 5, 7],
             'ZulipPython': [1, 2, 3],
             'API: Python': [1, 2, 3],
@@ -609,10 +54,9 @@ class TestMapArrays(ZulipTestCase):
        self.assertEqual(result,
                         {'Old desktop app': [32, 36, 39],
                          'Old iOS app': [1, 2, 3],
-                          'Desktop app': [2, 5, 7],
-                          'Mobile app': [1, 5, 7],
+                          'New iOS app': [1, 5, 7],
                          'Website': [1, 2, 3],
                          'Python API': [2, 4, 6],
                          'SomethingRandom': [4, 5, 6],
                          'GitHub webhook': [7, 7, 9],
-                          'Old Android app': [64, 63, 65]})
+                          'Android app': [64, 63, 65]})
--- a/analytics/urls.py
+++ b/analytics/urls.py
@@ -1,30 +1,17 @@
-from django.conf.urls import include, url
+from django.conf.urls import url, include
+from zerver.lib.rest import rest_dispatch

 import analytics.views
-from zerver.lib.rest import rest_dispatch

 i18n_urlpatterns = [
    # Server admin (user_profile.is_staff) visible stats pages
    url(r'^activity$', analytics.views.get_activity,
        name='analytics.views.get_activity'),
-    url(r'^activity/support$', analytics.views.support,
-        name='analytics.views.support'),
    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
        name='analytics.views.get_realm_activity'),
    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
        name='analytics.views.get_user_activity'),

-    url(r'^stats/realm/(?P<realm_str>[\S]+)/$', analytics.views.stats_for_realm,
-        name='analytics.views.stats_for_realm'),
-    url(r'^stats/installation$', analytics.views.stats_for_installation,
-        name='analytics.views.stats_for_installation'),
-    url(r'^stats/remote/(?P<remote_server_id>[\S]+)/installation$',
-        analytics.views.stats_for_remote_installation,
-        name='analytics.views.stats_for_remote_installation'),
-    url(r'^stats/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)/$',
-        analytics.views.stats_for_remote_realm,
-        name='analytics.views.stats_for_remote_realm'),
-
    # User-visible stats page
    url(r'^stats$', analytics.views.stats,
        name='analytics.views.stats'),
@@ -42,15 +29,6 @@ v1_api_and_json_patterns = [
    # get data for the graphs at /stats
    url(r'^analytics/chart_data$', rest_dispatch,
        {'GET': 'analytics.views.get_chart_data'}),
-    url(r'^analytics/chart_data/realm/(?P<realm_str>[\S]+)$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_realm'}),
-    url(r'^analytics/chart_data/installation$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_installation'}),
-    url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/installation$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_remote_installation'}),
-    url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)$',
-        rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_remote_realm'}),
 ]

 i18n_urlpatterns += [
--- a/analytics/views.py
+++ b/analytics/views.py
--- a/api/MANIFEST.in
+++ b/api/MANIFEST.in
@@ -0,0 +1,11 @@
+recursive-include integrations *
+include README.md
+include examples/zuliprc
+include examples/send-message
+include examples/subscribe
+include examples/get-public-streams
+include examples/unsubscribe
+include examples/list-members
+include examples/list-subscriptions
+include examples/print-messages
+include examples/recent-messages
--- a/api/README.md
+++ b/api/README.md
@@ -0,0 +1,172 @@
+#### Dependencies
+
+The [Zulip API](https://zulipchat.com/api) Python bindings require the
+following Python libraries:
+
+* requests (version >= 0.12.1)
+* simplejson
+* six
+* typing (version >= 3.5.2.2)
+
+#### Installing
+
+This package uses distutils, so you can just run:
+
+    python setup.py install
+
+#### Using the API
+
+For now, the only fully supported API operation is sending a message.
+The other API queries work, but are under active development, so
+please make sure we know you're using them so that we can notify you
+as we make any changes to them.
+
+The easiest way to use these API bindings is to base your tools off
+of the example tools under examples/ in this distribution.
+
+If you place your API key in the config file `~/.zuliprc` the Python
+API bindings will automatically read it in. The format of the config
+file is as follows:
+
+    [api]
+    key=<api key from the web interface>
+    email=<your email address>
+    site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
+
+Alternatively, you may explicitly use "--user", "--api-key", and
+`--site` in our examples, which is especially useful when testing.  If
+you are running several bots which share a home directory, we
+recommend using `--config` to specify the path to the `zuliprc` file
+for a specific bot.
+
+The command line equivalents for other configuration options are:
+
+    --insecure
+    --cert-bundle=<file>
+
+You can obtain your Zulip API key, create bots, and manage bots all
+from your Zulip settings page; with current Zulip there's also a
+button to download a `zuliprc` file for your account/server pair.
+
+A typical simple bot sending API messages will look as follows:
+
+At the top of the file:
+
+    # Make sure the Zulip API distribution's root directory is in sys.path, then:
+    import zulip
+    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
+
+When you want to send a message:
+
+    message = {
+      "type": "stream",
+      "to": ["support"],
+      "subject": "your subject",
+      "content": "your content",
+    }
+    zulip_client.send_message(message)
+
+If you are parsing arguments, you may find it useful to use Zulip's
+option group; see any of our API examples for details on how to do this.
+
+Additional examples:
+
+    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
+                         'subject': 'feedback', 'to': ['support']})
+    client.send_message({'type': 'private', 'content': 'Zulip rules!',
+                         'to': ['user1@example.com', 'user2@example.com']})
+
+send_message() returns a dict guaranteed to contain the following
+keys: msg, result.  For successful calls, result will be "success" and
+msg will be the empty string.  On error, result will be "error" and
+msg will describe what went wrong.
+
+#### Examples
+
+The API bindings package comes with several nice example scripts that
+show how to use the APIs; they are installed as part of the API
+bindings bundle.
+
+#### Logging
+
+The Zulip API comes with a ZulipStream class which can be used with the
+logging module:
+
+```
+import zulip
+import logging
+stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
+logger = logging.getLogger("your_logger")
+logger.addHandler(logging.StreamHandler(stream))
+logger.setLevel(logging.DEBUG)
+logger.info("This is an INFO test.")
+logger.debug("This is a DEBUG test.")
+logger.warn("This is a WARN test.")
+logger.error("This is a ERROR test.")
+```
+
+#### Sending messages
+
+You can use the included `zulip-send` script to send messages via the
+API directly from existing scripts.
+
+    zulip-send hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+Alternatively, if you don't want to use your ~/.zuliprc file:
+
+    zulip-send --user shakespeare-bot@example.com \
+        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+        --site https://zulip.example.com \
+        hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.
--- a/api/bin/zulip-send
+++ b/api/bin/zulip-send
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# zulip-send -- Sends a message to the specified recipients.
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import os
+import optparse
+import logging
+
+from typing import Any, Dict, List, Optional
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+import zulip
+
+logging.basicConfig()
+
+log = logging.getLogger('zulip-send')
+
+def do_send_message(client, message_data):
+    # type: (zulip.Client, Dict[str, Any]) -> bool
+    '''Sends a message and optionally prints status about the same.'''
+
+    if message_data['type'] == 'stream':
+        log.info('Sending message to stream "%s", subject "%s"... ' %
+                 (message_data['to'], message_data['subject']))
+    else:
+        log.info('Sending message to %s... ' % message_data['to'])
+    response = client.send_message(message_data)
+    if response['result'] == 'success':
+        log.info('Message sent.')
+        return True
+    else:
+        log.error(response['msg'])
+        return False
+
+def main(argv=None):
+    # type: (Optional[List[str]]) -> int
+    if argv is None:
+        argv = sys.argv
+
+    usage = """%prog [options] [recipient...]
+
+    Sends a message specified recipients.
+
+    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
+              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
+
+    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
+    '--user' and '--api-key' arguments.
+    """
+
+    parser = optparse.OptionParser(usage=usage)
+
+    # Grab parser options from the API common set
+    parser.add_option_group(zulip.generate_option_group(parser))
+
+    parser.add_option('-m', '--message',
+                      help='Specifies the message to send, prevents interactive prompting.')
+
+    group = optparse.OptionGroup(parser, 'Stream parameters')
+    group.add_option('-s', '--stream',
+                     dest='stream',
+                     action='store',
+                     help='Allows the user to specify a stream for the message.')
+    group.add_option('-S', '--subject',
+                     dest='subject',
+                     action='store',
+                     help='Allows the user to specify a subject for the message.')
+    parser.add_option_group(group)
+
+    (options, recipients) = parser.parse_args(argv[1:])
+
+    if options.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+    # Sanity check user data
+    if len(recipients) != 0 and (options.stream or options.subject):
+        parser.error('You cannot specify both a username and a stream/subject.')
+    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
+        parser.error('Stream messages must have a subject')
+    if len(recipients) == 0 and not (options.stream and options.subject):
+        parser.error('You must specify a stream/subject or at least one recipient.')
+
+    client = zulip.init_from_options(options)
+
+    if not options.message:
+        options.message = sys.stdin.read()
+
+    if options.stream:
+        message_data = {
+            'type': 'stream',
+            'content': options.message,
+            'subject': options.subject,
+            'to': options.stream,
+        }
+    else:
+        message_data = {
+            'type': 'private',
+            'content': options.message,
+            'to': recipients,
+        }
+
+    if not do_send_message(client, message_data):
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())
--- a/api/examples/create-user
+++ b/api/examples/create-user
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+from os import path
+import optparse
+
+usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
+
+Create a user. You must be a realm admin to use this API, and the user
+will be created in your realm.
+
+Example: create-user --site=http://localhost:9991 --user=rwbarton@example.com --new-email=jarthur@example.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
+"""
+
+sys.path.append(path.join(path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--new-email')
+parser.add_option('--new-password')
+parser.add_option('--new-full-name')
+parser.add_option('--new-short-name')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.create_user({
+    'email': options.new_email,
+    'password': options.new_password,
+    'full_name': options.new_full_name,
+    'short_name': options.new_short_name
+}))
--- a/api/examples/edit-message
+++ b/api/examples/edit-message
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
+
+Edits a message that you sent
+
+Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--message-id', default="")
+parser.add_option('--subject', default="")
+parser.add_option('--content', default="")
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "message_id": options.message_id,
+}
+if options.subject != "":
+    message_data["subject"] = options.subject
+if options.content != "":
+    message_data["content"] = options.content
+print(client.update_message(message_data))
--- a/api/examples/get-public-streams
+++ b/api/examples/get-public-streams
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out all the public streams in the realm.
+
+Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_streams(include_public=True, include_subscribed=False))
--- a/api/examples/list-members
+++ b/api/examples/list-members
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
+
+List the names and e-mail addresses of the people in your realm.
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+for user in client.get_members()["members"]:
+    print(user["full_name"], user["email"])
--- a/api/examples/list-subscriptions
+++ b/api/examples/list-subscriptions
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out a list of the user's subscriptions.
+
+Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.list_subscriptions())
--- a/api/examples/print-events
+++ b/api/examples/print-events
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out certain events received by the indicated bot or user matching the filter below.
+
+Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_event(event):
+    # type: (Dict[str, Any]) -> None
+    print(event)
+
+# This is a blocking call, and will continuously poll for new events
+# Note also the filter here is messages to the stream Denmark; if you
+# don't specify event_types it'll print all events.
+client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])
--- a/api/examples/print-messages
+++ b/api/examples/print-messages
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out each message received by the indicated bot or user.
+
+Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_message(message):
+    # type: (Dict[str, Any]) -> None
+    print(message)
+
+# This is a blocking call, and will continuously poll for new messages
+client.call_on_each_message(print_message)
--- a/api/examples/recent-messages
+++ b/api/examples/recent-messages
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages received by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+request = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.call_endpoint(
+    url='messages',
+    method='GET',
+    request=request,
+)
+
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])
--- a/api/examples/send-message
+++ b/api/examples/send-message
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
+
+Sends a test message to the specified recipients.
+
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--subject', default="test")
+parser.add_option('--message', default="test message")
+parser.add_option('--type', default='private')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if len(args) == 0:
+    parser.error("You must specify recipients")
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "type": options.type,
+    "content": options.message,
+    "subject": options.subject,
+    "to": args,
+}
+print(client.send_message(message_data))
--- a/api/examples/subscribe
+++ b/api/examples/subscribe
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is subscribed to the listed streams.
+
+Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))
--- a/api/examples/unsubscribe
+++ b/api/examples/unsubscribe
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is not subscribed to the listed streams.
+
+Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.remove_subscriptions(options.streams.split()))
--- a/api/examples/upload-file
+++ b/api/examples/upload-file
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import importlib
+import optparse
+import sys
+
+from six.moves import StringIO as _StringIO
+sys.path.insert(0, './api')
+from typing import IO
+import zulip
+
+class StringIO(_StringIO):
+    name = '' # https://github.com/python/typeshed/issues/598
+
+usage = """upload-file --user=<user's email address> --api-key=<user's api key> [options]
+
+Upload a file, and print the corresponding URI.
+
+Example: upload-file --user=cordelia@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --file-path=cat.png
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+If no --file-path is specified, a placeholder text file will be used instead.
+"""
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--file-path')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+file = None # type: IO
+if options.file_path:
+    file = open(options.file_path, 'rb')
+else:
+    file = StringIO('This is a test file.')
+    file.name = 'test.txt'
+
+response = client.upload_file(file)
+
+try:
+    print('File URI: {}'.format(response['uri']))
+except KeyError:
+    print('Error! API response was: {}'.format(response))
--- a/api/examples/zuliprc
+++ b/api/examples/zuliprc
@@ -0,0 +1,4 @@
+; Save this file as ~/.zuliprc
+[api]
+key=<your bot's api key from the web interface>
+email=<your bot's email address>
--- a/api/integrations/asana/zulip_asana_config.py
+++ b/api/integrations/asana/zulip_asana_config.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+### REQUIRED CONFIGURATION ###
+
+# Change these values to your Asana credentials.
+ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# Change these values to the credentials for your Asana bot.
+ZULIP_USER = "asana-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The Zulip stream that will receive Asana task updates.
+ZULIP_STREAM_NAME = "asana"
+
+
+### OPTIONAL CONFIGURATION ###
+
+# Set to None for logging to stdout when testing, and to a file for
+# logging in production.
+#LOG_FILE = "/var/tmp/zulip_asana.log"
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_asana.state"
+
+# When initially started, how many hours of messages to include.
+ASANA_INITIAL_HISTORY_HOURS = 1
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None
--- a/api/integrations/asana/zulip_asana_mirror
+++ b/api/integrations/asana/zulip_asana_mirror
@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Asana integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_asana_mirror" script is run continuously, possibly on a work computer
+# or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+import base64
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional, Any, Tuple
+
+import json
+import logging
+import os
+import time
+from six.moves import urllib
+from six.moves.urllib import request as urllib_request
+import sys
+
+
+try:
+    import dateutil.parser
+    from dateutil.tz import gettz
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_asana_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+if config.LOG_FILE:
+    logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+else:
+    logging.basicConfig(level=logging.INFO)
+
+client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
+                      site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
+
+def fetch_from_asana(path):
+    # type: (str) -> Optional[Dict[str, Any]]
+    """
+    Request a resource through the Asana API, authenticating using
+    HTTP basic auth.
+    """
+    auth = base64.encodestring(b'%s:' % (config.ASANA_API_KEY,))
+    headers = {"Authorization": "Basic %s" % auth}
+
+    url = "https://app.asana.com/api/1.0" + path
+    request = urllib_request.Request(url, None, headers) # type: ignore
+    result = urllib_request.urlopen(request) # type: ignore
+
+    return json.load(result)
+
+def send_zulip(topic, content):
+    # type: (str, str) -> Dict[str, str]
+    """
+    Send a message to Zulip using the configured stream and bot credentials.
+    """
+    message = {"type": "stream",
+               "sender": config.ZULIP_USER,
+               "to": config.ZULIP_STREAM_NAME,
+               "subject": topic,
+               "content": content,
+               }
+    return client.send_message(message)
+
+def datestring_to_datetime(datestring):
+    # type: (str) -> datetime
+    """
+    Given an ISO 8601 datestring, return the corresponding datetime object.
+    """
+    return dateutil.parser.parse(datestring).replace(
+        tzinfo=gettz('Z'))
+
+class TaskDict(dict):
+    """
+    A helper class to turn a dictionary with task information into an
+    object where each of the keys is an attribute for easy access.
+    """
+    def __getattr__(self, field):
+        # type: (TaskDict, str) -> Any
+        return self.get(field)
+
+def format_topic(task, projects):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string that will be the Zulip message topic for this task.
+    """
+    # Tasks can be associated with multiple projects, but in practice they seem
+    # to mostly be associated with one.
+    project_name = projects[task.projects[0]["id"]]
+    return "%s: %s" % (project_name, task.name)
+
+def format_assignee(task, users):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string describing the task's assignee.
+    """
+    if task.assignee:
+        assignee_name = users[task.assignee["id"]]
+        assignee_info = "**Assigned to**: %s (%s)" % (
+            assignee_name, task.assignee_status)
+    else:
+        assignee_info = "**Status**: Unassigned"
+
+    return assignee_info
+
+def format_due_date(task):
+    # type: (TaskDict) -> str
+    """
+    Return a string describing the task's due date.
+    """
+    if task.due_on:
+        due_date_info = "**Due on**: %s" % (task.due_on,)
+    else:
+        due_date_info = "**Due date**: None"
+    return due_date_info
+
+def format_task_creation_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a newly-created task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** created:
+
+~~~ quote
+%s
+~~~
+
+%s
+%s
+""" % (task.name, task.notes, assignee_info, due_date_info)
+    return topic, content
+
+def format_task_completion_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a completed task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** completed. :white_check_mark:
+
+%s
+%s
+""" % (task.name, assignee_info, due_date_info)
+    return topic, content
+
+def since():
+    # type: () -> datetime
+    """
+    Return a newness threshold for task events to be processed.
+    """
+    # If we have a record of the last event processed and it is recent, use it,
+    # else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
+    def default_since():
+        # type: () -> datetime
+        return datetime.utcnow() - timedelta(
+            hours=config.ASANA_INITIAL_HISTORY_HOURS)
+
+    if os.path.exists(config.RESUME_FILE):
+        try:
+            with open(config.RESUME_FILE, "r") as f:
+                datestring = f.readline().strip()
+                timestamp = float(datestring)
+                max_timestamp_processed = datetime.fromtimestamp(timestamp)
+                logging.info("Reading from resume file: " + datestring)
+        except (ValueError, IOError) as e:
+            logging.warn("Could not open resume file: " + str(e))
+            max_timestamp_processed = default_since()
+    else:
+        logging.info("No resume file, processing an initial history.")
+        max_timestamp_processed = default_since()
+
+    # Even if we can read a timestamp from RESUME_FILE, if it is old don't use
+    # it.
+    return max(max_timestamp_processed, default_since())
+
+def process_new_events():
+    # type: () -> None
+    """
+    Forward new Asana task events to Zulip.
+    """
+    # In task queries, Asana only exposes IDs for projects and users, so we need
+    # to look up the mappings.
+    projects = dict((elt["id"], elt["name"]) for elt in
+                    fetch_from_asana("/projects")["data"])
+    users = dict((elt["id"], elt["name"]) for elt in
+                 fetch_from_asana("/users")["data"])
+
+    cutoff = since()
+    max_timestamp_processed = cutoff
+    time_operations = (("created_at", format_task_creation_event),
+                       ("completed_at", format_task_completion_event))
+    task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
+                   "modified_at", "due_on", "name", "notes", "projects"]
+
+    # First, gather all of the tasks that need processing. We'll
+    # process them in order.
+    new_events = []
+
+    for project_id in projects:
+        project_url = "/projects/%d/tasks?opt_fields=%s" % (
+            project_id, ",".join(task_fields))
+        tasks = fetch_from_asana(project_url)["data"]
+
+        for task in tasks:
+            task = TaskDict(task)
+
+            for time_field, operation in time_operations:
+                if task[time_field]:
+                    operation_time = datestring_to_datetime(task[time_field])
+                    if operation_time > cutoff:
+                        new_events.append((operation_time, time_field, operation, task))
+
+    new_events.sort()
+    now = datetime.utcnow()
+
+    for operation_time, time_field, operation, task in new_events:
+        # Unfortunately, creating an Asana task is not an atomic operation. If
+        # the task was just created, or is missing basic information, it is
+        # probably because the task is still being filled out -- wait until the
+        # next round to process it.
+        if (time_field == "created_at") and \
+                (now - operation_time < timedelta(seconds=30)):
+            # The task was just created, give the user some time to fill out
+            # more information.
+            return
+
+        if (time_field == "created_at") and (not task.name) and \
+                (now - operation_time < timedelta(seconds=60)):
+            # If this new task hasn't had a name for a full 30 seconds, assume
+            # you don't plan on giving it one.
+            return
+
+        topic, content = operation(task, projects, users)
+        logging.info("Sending Zulip for " + topic)
+        result = send_zulip(topic, content)
+
+        # If the Zulip wasn't sent successfully, don't update the
+        # max timestamp processed so the task has another change to
+        # be forwarded. Exit, giving temporary issues time to
+        # resolve.
+        if not result.get("result"):
+            logging.warn("Malformed result, exiting:")
+            logging.warn(str(result))
+            return
+
+        if result["result"] != "success":
+            logging.warn(result["msg"])
+            return
+
+        if operation_time > max_timestamp_processed:
+            max_timestamp_processed = operation_time
+
+    if max_timestamp_processed > cutoff:
+        max_datestring = max_timestamp_processed.strftime("%s.%f")
+        logging.info("Updating resume file: " + max_datestring)
+        open(config.RESUME_FILE, 'w').write(max_datestring)
+
+while True:
+    try:
+        process_new_events()
+        time.sleep(5)
+    except KeyboardInterrupt:
+        logging.info("Shutting down...")
+        logging.info("Set LOG_FILE to log to a file instead of stdout.")
+        break
--- a/api/integrations/basecamp/zulip_basecamp_config.py
+++ b/api/integrations/basecamp/zulip_basecamp_config.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for basecamp account
+BASECAMP_ACCOUNT_ID = "12345678"
+BASECAMP_USERNAME = "foo@example.com"
+BASECAMP_PASSWORD = "p455w0rd"
+
+# This script will mirror this many hours of history on the first run.
+# On subsequent runs this value is ignored.
+BASECAMP_INITIAL_HISTORY_HOURS = 0
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "basecamp-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+ZULIP_STREAM_NAME = "basecamp"
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_basecamp.state"
--- a/api/integrations/basecamp/zulip_basecamp_mirror
+++ b/api/integrations/basecamp/zulip_basecamp_mirror
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Basecamp activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "basecamp-mirror.py" script is run continuously, possibly on a work computer
+# or preferably on a server.
+# You may need to install the python-requests library.
+
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import re
+import sys
+from stderror import write
+import os
+from datetime import datetime, timedelta
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_basecamp_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+from six.moves.html_parser import HTMLParser
+from typing import Any, Dict
+import six
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipBasecamp/" + VERSION)
+user_agent = "Basecamp To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+htmlParser = HTMLParser()
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # type: () -> None
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr.write("Could not open up log for writing:")
+            sys.stderr.write(str(e))
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE),)
+        sys.stderr.write(str(e))
+
+# builds the message dict for sending a message with the Zulip API
+def build_message(event):
+    # type: (Dict[str, Any]) -> Dict[str, Any]
+    if not ('bucket' in event and 'creator' in event and 'html_url' in event):
+        logging.error("Perhaps the Basecamp API changed behavior? "
+                      "This event doesn't have the expected format:\n%s" % (event,))
+        return None
+    # adjust the topic length to be bounded to 60 characters
+    topic = event['bucket']['name']
+    if len(topic) > 60:
+        topic = topic[0:57] + "..."
+    # get the action and target values
+    action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
+    target = htmlParser.unescape(event.get('target', ''))
+    # Some events have "excerpts", which we blockquote
+    excerpt = htmlParser.unescape(event.get('excerpt', ''))
+    if excerpt.strip() == "":
+        message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
+    else:
+        message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
+    # assemble the message data dict
+    message_data = {
+        "type": "stream",
+        "to": config.ZULIP_STREAM_NAME,
+        "subject": topic,
+        "content": message,
+    }
+    return message_data
+
+# the main run loop for this mirror script
+def run_mirror():
+    # type: () -> None
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    try:
+        with open(config.RESUME_FILE) as f:
+            since = f.read() # type: Any
+        since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
+        assert since, "resume file does not meet expected format"
+        since = since.string
+    except (AssertionError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (e,))
+        since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
+    try:
+        # we use an exponential backoff approach when we get 429 (Too Many Requests).
+        sleepInterval = 1
+        while True:
+            time.sleep(sleepInterval)
+            response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
+                                    params={'since': since},
+                                    auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
+                                    headers = {"User-Agent": user_agent})
+            if response.status_code == 200:
+                sleepInterval = 1
+                events = json.loads(response.text)
+                if len(events):
+                    logging.info("Got event(s): %s" % (response.text,))
+            if response.status_code >= 500:
+                logging.error(str(response.status_code))
+                continue
+            if response.status_code == 429:
+                # exponential backoff
+                sleepInterval *= 2
+                logging.error(str(response.status_code))
+                continue
+            if response.status_code == 400:
+                logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
+                sys.exit(-1)
+            if response.status_code == 401:
+                logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
+                sys.exit(-1)
+            if len(events):
+                since = events[0]['created_at']
+            for event in reversed(events):
+                message_data = build_message(event)
+                if not message_data:
+                    continue
+                zulip_api_result = client.send_message(message_data)
+                if zulip_api_result['result'] == "success":
+                    logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
+                else:
+                    logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
+                # update 'since' each time in case we get KeyboardInterrupted
+                since = event['created_at']
+                # avoid hitting rate-limit
+                time.sleep(0.2)
+
+    except KeyboardInterrupt:
+        logging.info("Shutting down, please hold")
+        open("events.last", 'w').write(since)
+        logging.info("Done!")
+
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr.write("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    run_mirror()
--- a/api/integrations/codebase/zulip_codebase_config.py
+++ b/api/integrations/codebase/zulip_codebase_config.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for your codebase account
+# Note that this is the Codebase API Username, found in the Settings page
+# for your account
+CODEBASE_API_USERNAME = "foo@example.com"
+CODEBASE_API_KEY = "1234561234567abcdef"
+
+# The URL of your codebase setup
+CODEBASE_ROOT_URL = "https://YOUR_COMPANY.codebasehq.com"
+
+# When initially started, how many hours of messages to include.
+# Note that the Codebase API only returns the 20 latest events,
+# if you have more than 20 events that fit within this window,
+# earlier ones may be lost
+CODEBASE_INITIAL_HISTORY_HOURS = 12
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "codebase-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The streams to send commit information and ticket information to
+ZULIP_COMMITS_STREAM_NAME = "codebase"
+ZULIP_TICKETS_STREAM_NAME = "tickets"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_codebase.state"
--- a/api/integrations/codebase/zulip_codebase_mirror
+++ b/api/integrations/codebase/zulip_codebase_mirror
@@ -0,0 +1,332 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Codebase HQ activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_codebase_mirror" script is run continuously, possibly on a work
+# computer or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import sys
+import os
+
+from datetime import datetime, timedelta
+
+try:
+    import dateutil.parser
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_codebase_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import six
+import zulip
+from typing import Any, List, Dict, Optional
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipCodebase/" + VERSION)
+user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+def make_api_call(path):
+    # type: (str) -> Optional[List[Dict[str, Any]]]
+    response = requests.get("https://api3.codebasehq.com/%s" % (path,),
+                            auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
+                            params={'raw': True},
+                            headers = {"User-Agent": user_agent,
+                                       "Content-Type": "application/json",
+                                       "Accept": "application/json"})
+    if response.status_code == 200:
+        return json.loads(response.text)
+
+    if response.status_code >= 500:
+        logging.error(str(response.status_code))
+        return None
+    if response.status_code == 403:
+        logging.error("Bad authorization from Codebase. Please check your credentials")
+        sys.exit(-1)
+    else:
+        logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
+        return None
+
+def make_url(path):
+    # type: (str) -> str
+    return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
+
+def handle_event(event):
+    # type: (Dict[str, Any]) -> None
+    event = event['event']
+    event_type = event['type']
+    actor_name = event['actor_name']
+
+    raw_props = event.get('raw_properties', {})
+
+    project_link = raw_props.get('project_permalink')
+
+    subject = None
+    content = None
+    if event_type == 'repository_creation':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        project_name = raw_props.get('name')
+        project_repo_type = raw_props.get('scm_type')
+
+        url = make_url("projects/%s" % (project_link,))
+        scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
+
+        subject = "Repository %s Created" % (project_name,)
+        content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
+    elif event_type == 'push':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        num_commits = raw_props.get('commits_count')
+        branch = raw_props.get('ref_name')
+        project = raw_props.get('project_name')
+        repo_link = raw_props.get('repository_permalink')
+        deleted_ref = raw_props.get('deleted_ref')
+        new_ref = raw_props.get('new_ref')
+
+        subject = "Push to %s on %s" % (branch, project)
+
+        if deleted_ref:
+            content = "%s deleted branch %s from %s" % (actor_name, branch, project)
+        else:
+            if new_ref:
+                branch = "new branch %s" % (branch,)
+            content = ("%s pushed %s commit(s) to %s in project %s:\n\n" %
+                       (actor_name, num_commits, branch, project))
+            for commit in raw_props.get('commits'):
+                ref = commit.get('ref')
+                url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
+                message = commit.get('message')
+                content += "* [%s](%s): %s\n" % (ref, url, message)
+    elif event_type == 'ticketing_ticket':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        assignee = raw_props.get('assignee')
+        priority = raw_props.get('priority')
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+
+        if assignee is None:
+            assignee = "no one"
+        subject = "#%s: %s" % (num, name)
+        content = ("""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" %
+                   (actor_name, num, url, priority, assignee, name))
+    elif event_type == 'ticketing_note':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        body = raw_props.get('content')
+        changes = raw_props.get('changes')
+
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+        subject = "#%s: %s" % (num, name)
+
+        content = ""
+        if body is not None and len(body) > 0:
+            content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
+
+        if 'status_id' in changes:
+            status_change = changes.get('status_id')
+            content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
+    elif event_type == 'ticketing_milestone':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        name = raw_props.get('name')
+        identifier = raw_props.get('identifier')
+        url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
+
+        subject = name
+        content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
+    elif event_type == 'comment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        comment = raw_props.get('content')
+        commit = raw_props.get('commit_ref')
+
+        # If there's a commit id, it's a comment to a commit
+        if commit:
+            repo_link = raw_props.get('repository_permalink')
+
+            url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
+
+            subject = "%s commented on %s" % (actor_name, commit)
+            content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
+        else:
+            # Otherwise, this is a Discussion item, and handle it
+            subj = raw_props.get("subject")
+            category = raw_props.get("category")
+            comment_content = raw_props.get("content")
+
+            subject = "Discussion: %s" % (subj,)
+
+            if category:
+                format_str = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~"
+                content = format_str % (actor_name, category, comment_content)
+            else:
+                content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
+
+    elif event_type == 'deployment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        start_ref = raw_props.get('start_ref')
+        end_ref = raw_props.get('end_ref')
+        environment = raw_props.get('environment')
+        servers = raw_props.get('servers')
+        repo_link = raw_props.get('repository_permalink')
+
+        start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
+        end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
+        between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (
+            project_link, repo_link, start_ref, end_ref))
+
+        subject = "Deployment to %s" % (environment,)
+
+        content = ("%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." %
+                   (actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment))
+        if servers is not None:
+            content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
+
+    elif event_type == 'named_tree':
+        # Docs say named_tree type used for new/deleting branches and tags,
+        # but experimental testing showed that they were all sent as 'push' events
+        pass
+    elif event_type == 'wiki_page':
+        logging.warn("Wiki page notifications not yet implemented")
+    elif event_type == 'sprint_creation':
+        logging.warn("Sprint notifications not yet implemented")
+    elif event_type == 'sprint_ended':
+        logging.warn("Sprint notifications not yet implemented")
+    else:
+        logging.info("Unknown event type %s, ignoring!" % (event_type,))
+
+    if subject and content:
+        if len(subject) > 60:
+            subject = subject[:57].rstrip() + '...'
+
+        res = client.send_message({"type": "stream",
+                                   "to": stream,
+                                   "subject": subject,
+                                   "content": content})
+        if res['result'] == 'success':
+            logging.info("Successfully sent Zulip with id: %s" % (res['id']))
+        else:
+            logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
+
+
+# the main run loop for this mirror script
+def run_mirror():
+    # type: () -> None
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    def default_since():
+        # type: () -> datetime
+        return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
+
+    try:
+        with open(config.RESUME_FILE) as f:
+            timestamp = f.read()
+        if timestamp == '':
+            since = default_since()
+        else:
+            since = datetime.fromtimestamp(float(timestamp))
+    except (ValueError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (str(e)))
+        since = default_since()
+
+    try:
+        sleepInterval = 1
+        while True:
+            events = make_api_call("activity")[::-1]
+            if events is not None:
+                sleepInterval = 1
+                for event in events:
+                    timestamp = event.get('event', {}).get('timestamp', '')
+                    event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None)
+                    if event_date > since:
+                        handle_event(event)
+                        since = event_date
+            else:
+                # back off a bit
+                if sleepInterval < 22:
+                    sleepInterval += 4
+            time.sleep(sleepInterval)
+
+    except KeyboardInterrupt:
+        open(config.RESUME_FILE, 'w').write(since.strftime("%s"))
+        logging.info("Shutting down Codebase mirror")
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # type: () -> None
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr.write("Could not open up log for writing:")
+            sys.stderr.write(str(e))
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
+        sys.stderr.write(str(e))
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr.write("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+    run_mirror()
--- a/api/integrations/git/post-receive
+++ b/api/integrations/git/post-receive
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-receive hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-receive" script is run after receive-pack has accepted a pack
+# and the repository has been updated.  It is passed arguments in through
+# stdin in the form
+#  <oldrev> <newrev> <refname>
+# For example:
+#  aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
+
+from __future__ import absolute_import
+from typing import Text
+import os
+import sys
+import subprocess
+import os.path
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_git_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipGit/" + VERSION)
+
+def git_repository_name():
+    # type: () -> Text
+    output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
+    if output.strip() == "true":
+        return os.path.basename(os.getcwd())[:-len(".git")]
+    else:
+        return os.path.basename(os.path.dirname(os.getcwd()))
+
+def git_commit_range(oldrev, newrev):
+    # type: (str, str) -> str
+    log_cmd = ["git", "log", "--reverse",
+               "--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
+    commits = ''
+    for ln in subprocess.check_output(log_cmd).splitlines():
+        author_email, commit_id, subject  = ln.split(None, 2)
+        if hasattr(config, "format_commit_message"):
+            commits += config.format_commit_message(author_email, subject, commit_id)
+        else:
+            commits += '!avatar(%s) %s\n' % (author_email, subject)
+    return commits
+
+def send_bot_message(oldrev, newrev, refname):
+    # type: (str, str, str) -> None
+    repo_name   = git_repository_name()
+    branch      = refname.replace('refs/heads/', '')
+    destination = config.commit_notice_destination(repo_name, branch, newrev)
+    if destination is None:
+        # Don't forward the notice anywhere
+        return
+
+    new_head = newrev[:12]
+    old_head = oldrev[:12]
+
+    if (oldrev == '0000000000000000000000000000000000000000' or
+            newrev == '0000000000000000000000000000000000000000'):
+        # New branch pushed or old branch removed
+        added = ''
+        removed = ''
+    else:
+        added   = git_commit_range(oldrev, newrev)
+        removed = git_commit_range(newrev, oldrev)
+
+    if oldrev == '0000000000000000000000000000000000000000':
+        message = '`%s` was pushed to new branch `%s`' % (new_head, branch)
+    elif newrev == '0000000000000000000000000000000000000000':
+        message = 'branch `%s` was removed (was `%s`)' % (branch, old_head)
+    elif removed:
+        message = '`%s` was pushed to `%s`, **REMOVING**:\n\n%s' % (new_head, branch, removed)
+        if added:
+            message += '\n**and adding**:\n\n' + added
+        message += '\n**A HISTORY REWRITE HAS OCCURRED!**'
+        message += '\n@everyone: Please check your local branches to deal with this.'
+    elif added:
+        message = '`%s` was deployed to `%s` with:\n\n%s' % (new_head, branch, added)
+    else:
+        message = '`%s` was pushed to `%s`... but nothing changed?' % (new_head, branch)
+
+    message_data = {
+        "type": "stream",
+        "to": destination["stream"],
+        "subject": destination["subject"],
+        "content": message,
+    }
+    client.send_message(message_data)
+
+for ln in sys.stdin:
+    oldrev, newrev, refname = ln.strip().split()
+    send_bot_message(oldrev, newrev, refname)
--- a/api/integrations/git/zulip_git_config.py
+++ b/api/integrations/git/zulip_git_config.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "git-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * repo   = the name of the git repository
+# * branch = the name of the branch that was pushed to
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit pushed to "master" to
+# * stream "commits"
+# * topic "master"
+# And similarly for branch "test-post-receive" (for use when testing).
+def commit_notice_destination(repo, branch, commit):
+    if branch in ["master", "test-post-receive"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (branch,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+# Modify this function to change how commits are displayed; the most
+# common customization is to include a link to the commit in your
+# graphical repository viewer, e.g.
+#
+# return '!avatar(%s) [%s](https://example.com/commits/%s)\n' % (author, subject, commit_id)
+def format_commit_message(author, subject, commit_id):
+    return '!avatar(%s) %s\n' % (author, subject)
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://zulip.example.com"
--- a/api/integrations/google/get-google-credentials
+++ b/api/integrations/google/get-google-credentials
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import datetime
+import httplib2
+import os
+
+from oauth2client import client
+from oauth2client import tools
+from oauth2client.file import Storage
+
+try:
+    import argparse
+    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
+except ImportError:
+    flags = None
+
+# If modifying these scopes, delete your previously saved credentials
+# at zulip/bots/gcal/
+# NOTE: When adding more scopes, add them after the previous one in the same field, with a space
+# seperating them.
+SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
+# This file contains the information that google uses to figure out which application is requesting
+# this client's data.
+CLIENT_SECRET_FILE = 'client_secret.json'
+APPLICATION_NAME = 'Zulip Calendar Bot'
+HOME_DIR = os.path.expanduser('~')
+
+def get_credentials():
+    # type: () -> client.Credentials
+    """Gets valid user credentials from storage.
+
+    If nothing has been stored, or if the stored credentials are invalid,
+    the OAuth2 flow is completed to obtain the new credentials.
+
+    Returns:
+        Credentials, the obtained credential.
+    """
+
+    credential_path = os.path.join(HOME_DIR,
+                                   'google-credentials.json')
+
+    store = Storage(credential_path)
+    credentials = store.get()
+    if not credentials or credentials.invalid:
+        flow = client.flow_from_clientsecrets(os.path.join(HOME_DIR, CLIENT_SECRET_FILE), SCOPES)
+        flow.user_agent = APPLICATION_NAME
+        if flags:
+            # This attempts to open an authorization page in the default web browser, and asks the user
+            # to grant the bot access to their data. If the user grants permission, the run_flow()
+            # function returns new credentials.
+            credentials = tools.run_flow(flow, store, flags)
+        else:  # Needed only for compatibility with Python 2.6
+            credentials = tools.run(flow, store)
+        print('Storing credentials to ' + credential_path)
+
+get_credentials()
--- a/api/integrations/google/google-calendar
+++ b/api/integrations/google/google-calendar
@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import datetime
+import httplib2
+import itertools
+import logging
+import optparse
+import os
+from six.moves import urllib
+import sys
+import time
+import traceback
+from typing import List, Set, Tuple, Iterable, Optional
+
+from oauth2client import client, tools
+from oauth2client.file import Storage
+try:
+    from googleapiclient import discovery
+except ImportError:
+    logging.exception('Install google-api-python-client')
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../'))
+import zulip
+
+SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
+CLIENT_SECRET_FILE = 'client_secret.json'
+APPLICATION_NAME = 'Zulip'
+HOME_DIR = os.path.expanduser('~')
+
+# Our cached view of the calendar, updated periodically.
+events = []  # type: List[Tuple[int, datetime.datetime, str]]
+
+# Unique keys for events we've already sent, so we don't remind twice.
+sent = set()  # type: Set[Tuple[int, datetime.datetime]]
+
+sys.path.append(os.path.dirname(__file__))
+
+parser = optparse.OptionParser(r"""
+
+%prog \
+    --user foo@zulip.com \
+    --calendar calendarID@example.calendar.google.com
+
+    This integration can be used to send yourself reminders, on Zulip, of Google Calendar Events.
+
+    Before running this integration make sure you run the get-google-credentials file to give Zulip
+    access to certain aspects of your Google Account.
+
+    This integration should be run on your local machine. Your API key and other information are
+    revealed to local users through the command line.
+
+    Depends on: google-api-python-client
+""")
+
+
+parser.add_option('--interval',
+                  dest='interval',
+                  default=30,
+                  type=int,
+                  action='store',
+                  help='Minutes before event for reminder [default: 30]',
+                  metavar='MINUTES')
+
+parser.add_option('--calendar',
+                  dest = 'calendarID',
+                  default = 'primary',
+                  type = str,
+                  action = 'store',
+                  help = 'Calendar ID for the calendar you want to receive reminders from.')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+
+(options, args) = parser.parse_args()
+
+if not (options.zulip_email):
+    parser.error('You must specify --user')
+
+zulip_client = zulip.init_from_options(options)
+
+def get_credentials():
+    # type: () -> client.Credentials
+    """Gets valid user credentials from storage.
+
+    If nothing has been stored, or if the stored credentials are invalid,
+    an exception is thrown and the user is informed to run the script in this directory to get
+    credentials.
+
+    Returns:
+        Credentials, the obtained credential.
+    """
+    try:
+        credential_path = os.path.join(HOME_DIR,
+                                       'google-credentials.json')
+
+        store = Storage(credential_path)
+        credentials = store.get()
+
+        return credentials
+    except client.Error:
+        logging.exception('Error while trying to open the `google-credentials.json` file.')
+    except IOError:
+        logging.error("Run the get-google-credentials script from this directory first.")
+
+
+def get_events():
+    # type: () -> Iterable[Tuple[int, datetime.datetime, str]]
+    credentials = get_credentials()
+    creds = credentials.authorize(httplib2.Http())
+    service = discovery.build('calendar', 'v3', http=creds)
+
+    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time
+    feed = service.events().list(calendarId=options.calendarID, timeMin=now, maxResults=5,
+                                 singleEvents=True, orderBy='startTime').execute()
+
+    for event in feed["items"]:
+        try:
+            start = event["start"]["dateTime"]
+        except KeyError:
+            start = event["start"]["date"]
+        start = start[:19]
+        # All-day events can have only a date
+        fmt = '%Y-%m-%dT%H:%M:%S' if 'T' in start else '%Y-%m-%d'
+        start = datetime.datetime.strptime(start, fmt)
+        try:
+            yield (event["id"], start, event["summary"])
+        except KeyError:
+            yield (event["id"], start, "(No Title)")
+
+
+def send_reminders():
+    # type: () -> Optional[None]
+    global sent
+
+    messages = []
+    keys = set()
+    now = datetime.datetime.now()
+
+    for id, start, summary in events:
+        dt = start - now
+        if dt.days == 0 and dt.seconds < 60 * options.interval:
+            # The unique key includes the start time, because of
+            # repeating events.
+            key = (id, start)
+            if key not in sent:
+                if start.hour == 0 and start.minute == 0:
+                    line = '%s is today.' % (summary,)
+                else:
+                    line = '%s starts at %s' % (summary, start.strftime('%H:%M'))
+                print('Sending reminder:', line)
+                messages.append(line)
+                keys.add(key)
+
+    if not messages:
+        return
+
+    if len(messages) == 1:
+        message = 'Reminder: ' + messages[0]
+    else:
+        message = 'Reminder:\n\n' + '\n'.join('* ' + m for m in messages)
+
+    zulip_client.send_message(dict(
+        type    = 'private',
+        to      = options.zulip_email,
+        sender  = options.zulip_email,
+        content = message))
+
+    sent.update(keys)
+
+# Loop forever
+for i in itertools.count():
+    try:
+        # We check reminders every minute, but only
+        # download the calendar every 10 minutes.
+        if not i % 10:
+            events = list(get_events())
+        send_reminders()
+    except:
+        logging.exception("Couldn't download Google calendar and/or couldn't post to Zulip.")
+    time.sleep(60)
--- a/api/integrations/hg/zulip-changegroup.py
+++ b/api/integrations/hg/zulip-changegroup.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip hook for Mercurial changeset pushes.
+# Copyright Â© 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+#
+# This hook is called when changesets are pushed to the master repository (ie
+# `hg push`). See https://zulipchat.com/integrations for installation instructions.
+from __future__ import absolute_import
+
+import zulip
+from six.moves import range
+from typing import Any, Optional, Text
+from mercurial import ui, repo
+
+VERSION = "0.9"
+
+def format_summary_line(web_url, user, base, tip, branch, node):
+    # type: (str, str, int, int, str, Text) -> Text
+    """
+    Format the first line of the message, which contains summary
+    information about the changeset and links to the changelog if a
+    web URL has been configured:
+
+    Jane Doe <jane@example.com> pushed 1 commit to master (170:e494a5be3393):
+    """
+    revcount = tip - base
+    plural = "s" if revcount > 1 else ""
+
+    if web_url:
+        shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
+        summary_url = "{shortlog}{tip}?revcount={revcount}".format(
+            shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
+        formatted_commit_count = "[{revcount} commit{s}]({url})".format(
+            revcount=revcount, s=plural, url=summary_url)
+    else:
+        formatted_commit_count = "{revcount} commit{s}".format(
+            revcount=revcount, s=plural)
+
+    return u"**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
+        user=user, commits=formatted_commit_count, branch=branch, tip=tip,
+        node=node[:12])
+
+def format_commit_lines(web_url, repo, base, tip):
+    # type: (str, repo, int, int) -> str
+    """
+    Format the per-commit information for the message, including the one-line
+    commit summary and a link to the diff if a web URL has been configured:
+    """
+    if web_url:
+        rev_base_url = web_url.rstrip("/") + "/rev/"
+
+    commit_summaries = []
+    for rev in range(base, tip):
+        rev_node = repo.changelog.node(rev)
+        rev_ctx = repo.changectx(rev_node)
+        one_liner = rev_ctx.description().split("\n")[0]
+
+        if web_url:
+            summary_url = rev_base_url + str(rev_ctx)
+            summary = "* [{summary}]({url})".format(
+                summary=one_liner, url=summary_url)
+        else:
+            summary = "* {summary}".format(summary=one_liner)
+
+        commit_summaries.append(summary)
+
+    return "\n".join(summary for summary in commit_summaries)
+
+def send_zulip(email, api_key, site, stream, subject, content):
+    # type: (str, str, str, str, str, Text) -> str
+    """
+    Send a message to Zulip using the provided credentials, which should be for
+    a bot in most cases.
+    """
+    client = zulip.Client(email=email, api_key=api_key,
+                          site=site,
+                          client="ZulipMercurial/" + VERSION)
+
+    message_data = {
+        "type": "stream",
+        "to": stream,
+        "subject": subject,
+        "content": content,
+    }
+
+    client.send_message(message_data)
+
+def get_config(ui, item):
+    # type: (ui, str) -> Optional[str]
+    try:
+        # configlist returns everything in lists.
+        return ui.configlist('zulip', item)[0]
+    except IndexError:
+        return None
+
+def hook(ui, repo, **kwargs):
+    # type: (ui, repo, Optional[Text]) -> None
+    """
+    Invoked by configuring a [hook] entry in .hg/hgrc.
+    """
+    hooktype = kwargs["hooktype"]
+    node = kwargs["node"]
+
+    ui.debug("Zulip: received {hooktype} event\n".format(hooktype=hooktype))
+
+    if hooktype != "changegroup":
+        ui.warn("Zulip: {hooktype} not supported\n".format(hooktype=hooktype))
+        exit(1)
+
+    ctx = repo.changectx(node)
+    branch = ctx.branch()
+
+    # If `branches` isn't specified, notify on all branches.
+    branch_whitelist = get_config(ui, "branches")
+    branch_blacklist = get_config(ui, "ignore_branches")
+
+    if branch_whitelist:
+        # Only send notifications on branches we are watching.
+        watched_branches = [b.lower().strip() for b in branch_whitelist.split(",")]
+        if branch.lower() not in watched_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    if branch_blacklist:
+        # Don't send notifications for branches we've ignored.
+        ignored_branches = [b.lower().strip() for b in branch_blacklist.split(",")]
+        if branch.lower() in ignored_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    # The first and final commits in the changeset.
+    base = repo[node].rev()
+    tip = len(repo)
+
+    email = get_config(ui, "email")
+    api_key = get_config(ui, "api_key")
+    site = get_config(ui, "site")
+
+    if not (email and api_key):
+        ui.warn("Zulip: missing email or api_key configurations\n")
+        ui.warn("in the [zulip] section of your .hg/hgrc.\n")
+        exit(1)
+
+    stream = get_config(ui, "stream")
+    # Give a default stream if one isn't provided.
+    if not stream:
+        stream = "commits"
+
+    web_url = get_config(ui, "web_url")
+    user = ctx.user()
+    content = format_summary_line(web_url, user, base, tip, branch, node)
+    content += format_commit_lines(web_url, repo, base, tip)
+
+    subject = branch
+
+    ui.debug("Sending to Zulip:\n")
+    ui.debug(content + "\n")
+
+    send_zulip(email, api_key, site, stream, subject, content)
--- a/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
+++ b/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
@@ -0,0 +1,149 @@
+/*
+* Copyright (c) 2014 Zulip, Inc
+*/
+
+package org.zulip.jira
+
+import static com.atlassian.jira.event.type.EventType.*
+
+import com.atlassian.jira.event.issue.AbstractIssueEventListener
+import com.atlassian.jira.event.issue.IssueEvent
+
+import java.util.logging.Level
+import java.util.logging.Logger
+
+import org.apache.commons.httpclient.HttpClient
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.methods.PostMethod
+import org.apache.commons.httpclient.NameValuePair
+
+class ZulipListener extends AbstractIssueEventListener {
+    Logger LOGGER = Logger.getLogger(ZulipListener.class.getName());
+
+    // The email address of one of the bots you created on your Zulip settings page.
+    String zulipEmail = ""
+    // That bot's API key.
+    String zulipAPIKey = ""
+
+    // What stream to send messages to. Must already exist.
+    String zulipStream = "jira"
+
+    // The base JIRA url for browsing
+    String issueBaseUrl = "https://jira.COMPANY.com/browse/"
+
+    // Your zulip domain
+    String base_url = "https://zulip.example.com/"
+
+    @Override
+    void workflowEvent(IssueEvent event) {
+      processIssueEvent(event)
+    }
+
+    String processIssueEvent(IssueEvent event) {
+      String author = event.user.displayName
+      String issueId = event.issue.key
+      String issueUrl = issueBaseUrl + issueId
+      String issueUrlMd = String.format("[%s](%s)", issueId, issueBaseUrl + issueId)
+      String title = event.issue.summary
+      String subject = truncate(String.format("%s: %s", issueId, title), 60)
+      String assignee = "no one"
+      if (event.issue.assignee) {
+        assignee = event.issue.assignee.name
+      }
+      String comment = "";
+      if (event.comment) {
+        comment = event.comment.body
+      }
+
+      String content;
+
+      // Event types:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/event/type/EventType.html
+      // Issue API:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/issue/Issue.html
+      switch (event.getEventTypeId()) {
+        case ISSUE_COMMENTED_ID:
+          content = String.format("%s **updated** %s with comment:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        case ISSUE_CREATED_ID:
+          content = String.format("%s **created** %s priority %s, assigned to @**%s**: \n\n> %s",
+                                  author, issueUrlMd, event.issue.priorityObject.name,
+                                  assignee, title)
+          break
+        case ISSUE_ASSIGNED_ID:
+          content = String.format("%s **reassigned** %s to **%s**",
+                                  author, issueUrlMd, assignee)
+          break
+        case ISSUE_DELETED_ID:
+          content = String.format("%s **deleted** %s!",
+                                  author, issueUrlMd)
+          break
+        case ISSUE_RESOLVED_ID:
+          content = String.format("%s **resolved** %s as %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_CLOSED_ID:
+          content = String.format("%s **closed** %s with resolution %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_REOPENED_ID:
+          content = String.format("%s **reopened** %s:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        default:
+          return
+      }
+
+      sendStreamMessage(zulipStream, subject, content)
+    }
+
+    String post(String method, NameValuePair[] parameters) {
+      PostMethod post = new PostMethod(zulipUrl(method))
+      post.setRequestHeader("Content-Type", post.FORM_URL_ENCODED_CONTENT_TYPE)
+      // TODO: Include more useful data in the User-agent
+      post.setRequestHeader("User-agent", "ZulipJira/0.1")
+      try {
+        post.setRequestBody(parameters)
+        HttpClient client = new HttpClient()
+        client.executeMethod(post)
+        String response = post.getResponseBodyAsString()
+        if (post.getStatusCode() != HttpStatus.SC_OK) {
+          String params = ""
+          for (NameValuePair pair: parameters) {
+              params += "\n" + pair.getName() + ":" + pair.getValue()
+          }
+          LOGGER.log(Level.SEVERE, "Error sending Zulip message:\n" + response + "\n\n" +
+                                   "We sent:" + params)
+        }
+        return response;
+      } catch (IOException e) {
+        throw new RuntimeException(e)
+      } finally {
+        post.releaseConnection()
+      }
+    }
+
+    String truncate(String string, int length) {
+      if (string.length() < length) {
+        return string
+      }
+      return string.substring(0, length - 3) + "..."
+    }
+
+    String sendStreamMessage(String stream, String subject, String message) {
+      NameValuePair[] body = [new NameValuePair("api-key", zulipAPIKey),
+                              new NameValuePair("email",   zulipEmail),
+                              new NameValuePair("type",    "stream"),
+                              new NameValuePair("to",      stream),
+                              new NameValuePair("subject", subject),
+                              new NameValuePair("content", message)]
+      return post("send_message", body);
+    }
+
+    String zulipUrl(method) {
+      return base_url.replaceAll("/+$", "") + "/api/v1/" + method
+    }
+}
--- a/api/integrations/nagios/nagios-notify-zulip
+++ b/api/integrations/nagios/nagios-notify-zulip
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+import optparse
+import zulip
+
+from typing import List, Text, Dict, Any
+
+VERSION = "0.9"
+# Nagios passes the notification details as command line options.
+# In Nagios, "output" means "first line of output", and "long
+# output" means "other lines of output".
+parser = optparse.OptionParser()  # type: optparse.OptionParser
+parser.add_option('--output', default='')
+parser.add_option('--long-output', default='')
+parser.add_option('--stream', default='nagios')
+parser.add_option('--config', default='/etc/nagios3/zuliprc')
+for opt in ('type', 'host', 'service', 'state'):
+    parser.add_option('--' + opt)
+(opts, args) = parser.parse_args()  # type: Any, List[Text]
+
+client = zulip.Client(config_file=opts.config,
+                      client="ZulipNagios/" + VERSION)  # type: zulip.Client
+
+msg = dict(type='stream', to=opts.stream)  # type: Dict[str, Any]
+
+# Set a subject based on the host or service in question.  This enables
+# threaded discussion of multiple concurrent issues, and provides useful
+# context when narrowed.
+#
+# We send PROBLEM and RECOVERY messages to the same subject.
+if opts.service is None:
+    # Host notification
+    thing = 'host'  # type: Text
+    msg['subject'] = 'host %s' % (opts.host,)
+else:
+    # Service notification
+    thing = 'service'
+    msg['subject'] = 'service %s on %s' % (opts.service, opts.host)
+
+if len(msg['subject']) > 60:
+    msg['subject'] = msg['subject'][0:57].rstrip() + "..."
+# e.g. **PROBLEM**: service is CRITICAL
+msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
+
+# The "long output" can contain newlines represented by "\n" escape sequences.
+# The Nagios mail command uses /usr/bin/printf "%b" to expand these.
+# We will be more conservative and handle just this one escape sequence.
+output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip()  # type: Text
+if output:
+    # Put any command output in a code block.
+    msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
+
+client.send_message(msg)
--- a/api/integrations/nagios/zulip_nagios.cfg
+++ b/api/integrations/nagios/zulip_nagios.cfg
@@ -0,0 +1,21 @@
+define contact{
+        contact_name                    zulip
+        alias                           zulip
+        service_notification_period     24x7
+        host_notification_period        24x7
+        service_notification_options    w,u,c,r
+        host_notification_options       d,r
+        service_notification_commands   notify-service-by-zulip
+        host_notification_commands      notify-host-by-zulip
+        }
+
+# Zulip commands
+define command {
+    command_name    notify-host-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
+}
+
+define command {
+    command_name    notify-service-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
+}
--- a/api/integrations/nagios/zuliprc.example
+++ b/api/integrations/nagios/zuliprc.example
@@ -0,0 +1,5 @@
+# Fill these values in with the appropriate values for your realm, and
+# then install this value at /etc/nagios3/zuliprc
+[api]
+email = nagios-bot@example.com
+key = 0123456789abcdef0123456789abcdef
--- a/api/integrations/openshift/post_deploy
+++ b/api/integrations/openshift/post_deploy
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-receive hook.
+# Copyright © 2012-2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_openshift_config as config
+VERSION = '0.1'
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client='ZulipOpenShift/' + VERSION)
+
+def get_deployment_details():
+    # type: () -> Dict[str, str]
+    # "gear deployments" output example:
+    #   Activation time - Deployment ID - Git Ref - Git SHA1
+    #   2017-01-07 15:40:30 -0500 - 9e2b7143 - master - b9ce57c - ACTIVE
+    dep = subprocess.check_output(['gear', 'deployments']).splitlines()[1]
+    splits = dep.split(' - ')
+
+    return dict(app_name=os.environ['OPENSHIFT_APP_NAME'],
+                url=os.environ['OPENSHIFT_APP_DNS'],
+                branch=splits[2],
+                commit_id=splits[3])
+
+def send_bot_message(deployment):
+    # type: (Dict[str, str]) -> None
+    destination = config.deployment_notice_destination(deployment['branch'])
+    if destination is None:
+        # No message should be sent
+        return
+    message = config.format_deployment_message(**deployment)
+
+    client.send_message({
+        'type': 'stream',
+        'to': destination['stream'],
+        'subject': destination['subject'],
+        'content': message,
+    })
+
+    return
+
+deployment = get_deployment_details()
+send_bot_message(deployment)
--- a/api/integrations/openshift/zulip_openshift_config.py
+++ b/api/integrations/openshift/zulip_openshift_config.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# https://github.com/python/mypy/issues/1141
+from typing import Text
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = 'openshift-bot@example.com'
+ZULIP_API_KEY = '0123456789abcdef0123456789abcdef'
+
+# deployment_notice_destination() lets you customize where deployment notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * branch = the name of the branch where the deployed commit was
+#            pushed to
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit pushed to "master" to
+# * stream "deployments"
+# * topic "master"
+# And similarly for branch "test-post-receive" (for use when testing).
+def deployment_notice_destination(branch):
+    # type: (str) -> Dict[str, Text]
+    if branch in ['master', 'test-post-receive']:
+        return dict(stream  = 'deployments',
+                    subject = u'%s' % (branch,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+# Modify this function to change how deployments are displayed
+#
+# It takes the following arguments:
+# * app_name  = the name of the app being deployed
+# * url       = the FQDN (Fully Qualified Domain Name) where the app
+#                can be found
+# * branch    = the name of the branch where the deployed commit was
+#                pushed to
+# * commit_id = hash of the commit that triggered the deployment
+# * dep_id    = deployment id
+# * dep_time  = deployment timestamp
+def format_deployment_message(
+        app_name='', url='', branch='', commit_id='', dep_id='', dep_time=''):
+    # type: (str, str, str, str, str, str) -> str
+    return 'Deployed commit `%s` (%s) in [%s](%s)' % (
+        commit_id, branch, app_name, url)
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None # type: str
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = 'https://zulip.example.com'
--- a/api/integrations/perforce/git_p4.py
+++ b/api/integrations/perforce/git_p4.py
--- a/api/integrations/perforce/license.txt
+++ b/api/integrations/perforce/license.txt
@@ -0,0 +1,26 @@
+git_p4.py was downloaded from https://raw.github.com/git/git/34022ba/git-p4.py
+
+The header of that file references <http://opensource.org/licenses/mit-license.php>,
+the textual contents of which are included below.
+
+The MIT License (MIT)
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- a/api/integrations/perforce/zulip_change-commit.py
+++ b/api/integrations/perforce/zulip_change-commit.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+'''Zulip notification change-commit hook.
+
+In Perforce, The "change-commit" trigger is fired after a metadata has been
+created, files have been transferred, and the changelist committed to the depot
+database.
+
+This specific trigger expects command-line arguments in the form:
+  %change% %changeroot%
+
+For example:
+  1234 //depot/security/src/
+
+'''
+from __future__ import print_function
+
+import os
+import sys
+import os.path
+
+import git_p4
+
+__version__ = "0.1"
+
+sys.path.insert(0, os.path.dirname(__file__))
+from typing import Any, Dict, Optional, Text
+import zulip_perforce_config as config
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipPerforce/" + __version__)  # type: zulip.Client
+
+try:
+    changelist = int(sys.argv[1])  # type: int
+    changeroot = sys.argv[2]  # type: str
+except IndexError:
+    print("Wrong number of arguments.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+except ValueError:
+    print("First argument must be an integer.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+
+metadata = git_p4.p4_describe(changelist)  # type: Dict[str, str]
+
+destination = config.commit_notice_destination(changeroot, changelist)  # type: Optional[Dict[str, str]]
+if destination is None:
+    # Don't forward the notice anywhere
+    sys.exit(0)
+
+message = "**{0}** committed revision @{1} to `{2}`.\n\n> {3}".format(
+    metadata["user"],
+    metadata["change"],
+    changeroot,
+    metadata["desc"])  # type: str
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}  # type: Dict[str, Any]
+client.send_message(message_data)
--- a/api/integrations/perforce/zulip_perforce_config.py
+++ b/api/integrations/perforce/zulip_perforce_config.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "p4-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+ZULIP_SITE = "https://zulip.example.com"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the Perforce depot on the server
+# * changelist = the changelist id
+#
+# Returns a dictionary encoding the stream and topic to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for ones in the
+# "master-plan" and "secret" subdirectories of //depot/ to:
+# * stream "depot_subdirectory-commits"
+# * subject "change_root"
+def commit_notice_destination(path, changelist):
+    dirs = path.split('/')
+    if len(dirs) >= 4 and dirs[3] not in ("*", "..."):
+        directory = dirs[3]
+    else:
+        # No subdirectory, so just use "depot"
+        directory = dirs[2]
+
+    if directory not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "%s-commits" % (directory,),
+                    subject = path)
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
--- a/api/integrations/rss/rss-bot
+++ b/api/integrations/rss/rss-bot
@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# RSS integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import calendar
+import errno
+import hashlib
+from six.moves.html_parser import HTMLParser
+import logging
+import optparse
+import os
+import sys
+import time
+from six.moves import urllib
+from typing import Dict, List, Tuple, Any
+
+import feedparser
+import zulip
+VERSION = "0.9" # type: str
+RSS_DATA_DIR = os.path.expanduser(os.path.join('~', '.cache', 'zulip-rss')) # type: str
+OLDNESS_THRESHOLD = 30 # type: int
+
+usage = """Usage: Send summaries of RSS entries for your favorite feeds to Zulip.
+
+This bot requires the feedparser module.
+
+To use this script:
+
+1. Create an RSS feed file containing 1 feed URL per line (default feed
+   file location: ~/.cache/zulip-rss/rss-feeds)
+2. Subscribe to the stream that will receive RSS updates (default stream: rss)
+3. create a ~/.zuliprc as described on https://zulipchat.com/api#api_keys
+4. Test the script by running it manually, like this:
+
+/usr/local/share/zulip/integrations/rss/rss-bot
+
+You can customize the location on the feed file and recipient stream, e.g.:
+
+/usr/local/share/zulip/integrations/rss/rss-bot --feed-file=/path/to/my-feeds --stream=my-rss-stream
+
+4. Configure a crontab entry for this script. A sample crontab entry for
+processing feeds stored in the default location and sending to the default
+stream every 5 minutes is:
+
+*/5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
+
+parser = optparse.OptionParser(usage) # type: optparse.OptionParser
+parser.add_option('--stream',
+                  dest='stream',
+                  help='The stream to which to send RSS messages.',
+                  default="rss",
+                  action='store')
+parser.add_option('--data-dir',
+                  dest='data_dir',
+                  help='The directory where feed metadata is stored',
+                  default=os.path.join(RSS_DATA_DIR),
+                  action='store')
+parser.add_option('--feed-file',
+                  dest='feed_file',
+                  help='The file containing a list of RSS feed URLs to follow, one URL per line',
+                  default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
+                  action='store')
+parser.add_option_group(zulip.generate_option_group(parser))
+(opts, args) = parser.parse_args() # type: Tuple[Any, List[str]]
+
+def mkdir_p(path):
+    # type: (str) -> None
+    # Python doesn't have an analog to `mkdir -p` < Python 3.2.
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+try:
+    mkdir_p(opts.data_dir)
+except OSError:
+    # We can't write to the logfile, so just print and give up.
+    print("Unable to store RSS data at %s." % (opts.data_dir,), file=sys.stderr)
+    exit(1)
+
+log_file = os.path.join(opts.data_dir, "rss-bot.log") # type: str
+log_format = "%(asctime)s: %(message)s" # type: str
+logging.basicConfig(format=log_format)
+
+formatter = logging.Formatter(log_format) # type: logging.Formatter
+file_handler = logging.FileHandler(log_file) # type: logging.FileHandler
+file_handler.setFormatter(formatter)
+
+logger = logging.getLogger(__name__) # type: logging.Logger
+logger.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+def log_error_and_exit(error):
+    # type: (str) -> None
+    logger.error(error)
+    logger.error(usage)
+    exit(1)
+
+class MLStripper(HTMLParser):
+    def __init__(self):
+        # type: () -> None
+        self.reset()
+        self.fed = [] # type: List[str]
+
+    def handle_data(self, data):
+        # type: (str) -> None
+        self.fed.append(data)
+
+    def get_data(self):
+        # type: () -> str
+        return ''.join(self.fed)
+
+def strip_tags(html):
+    # type: (str) -> str
+    stripper = MLStripper()
+    stripper.feed(html)
+    return stripper.get_data()
+
+def compute_entry_hash(entry):
+    # type: (Dict[str, Any]) -> str
+    entry_time = entry.get("published", entry.get("updated"))
+    entry_id = entry.get("id", entry.get("link"))
+    return hashlib.md5(entry_id + str(entry_time)).hexdigest()
+
+def elide_subject(subject):
+    # type: (str) -> str
+    MAX_TOPIC_LENGTH = 60
+    if len(subject) > MAX_TOPIC_LENGTH:
+        subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
+    return subject
+
+def send_zulip(entry, feed_name):
+    # type: (Any, str) -> Dict[str, Any]
+    content = "**[%s](%s)**\n%s\n%s" % (entry.title,
+                                        entry.link,
+                                        strip_tags(entry.summary),
+                                        entry.link) # type: str
+    message = {"type": "stream",
+               "sender": opts.zulip_email,
+               "to": opts.stream,
+               "subject": elide_subject(feed_name),
+               "content": content,
+               } # type: Dict[str, str]
+    return client.send_message(message)
+
+try:
+    with open(opts.feed_file, "r") as f:
+        feed_urls = [feed.strip() for feed in f.readlines()] # type: List[str]
+except IOError:
+    log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
+
+client = zulip.Client(email=opts.zulip_email, api_key=opts.zulip_api_key,
+                      site=opts.zulip_site, client="ZulipRSS/" + VERSION) # type: zulip.Client
+
+first_message = True # type: bool
+
+for feed_url in feed_urls:
+    feed_file = os.path.join(opts.data_dir, urllib.parse.urlparse(feed_url).netloc) # Type: str
+
+    try:
+        with open(feed_file, "r") as f:
+            old_feed_hashes = dict((line.strip(), True) for line in f.readlines()) # type: Dict[str, bool]
+    except IOError:
+        old_feed_hashes = {}
+
+    new_hashes = [] # type: List[str]
+    data = feedparser.parse(feed_url) # type: feedparser.parse
+
+    for entry in data.entries:
+        entry_hash = compute_entry_hash(entry) # type: str
+        # An entry has either been published or updated.
+        entry_time = entry.get("published_parsed", entry.get("updated_parsed")) # type: Tuple[int, int]
+        if entry_time is not None and (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
+            # As a safeguard against misbehaving feeds, don't try to process
+            # entries older than some threshold.
+            continue
+        if entry_hash in old_feed_hashes:
+            # We've already seen this. No need to process any older entries.
+            break
+        if (not old_feed_hashes) and (len(new_hashes) >= 3):
+            # On a first run, pick up the 3 most recent entries. An RSS feed has
+            # entries in reverse chronological order.
+            break
+
+        feed_name = data.feed.title or feed_url # type: str
+
+        response = send_zulip(entry, feed_name) # type: Dict[str, Any]
+        if response["result"] != "success":
+            logger.error("Error processing %s" % (feed_url,))
+            logger.error(str(response))
+            if first_message:
+                # This is probably some fundamental problem like the stream not
+                # existing or something being misconfigured, so bail instead of
+                # getting the same error for every RSS entry.
+                log_error_and_exit("Failed to process first message")
+        # Go ahead and move on -- perhaps this entry is corrupt.
+        new_hashes.append(entry_hash)
+        first_message = False
+
+    with open(feed_file, "a") as f:
+        for hash in new_hashes:
+            f.write(hash + "\n")
+
+    logger.info("Sent zulips for %d %s entries" % (len(new_hashes), feed_url))
--- a/api/integrations/svn/post-commit
+++ b/api/integrations/svn/post-commit
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-commit hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-commit" script is run after a transaction is completed and a new
+# revision is created.  It is passed arguments on the command line in this
+# form:
+#  <path> <revision>
+# For example:
+# /srv/svn/carols 1843
+
+import os
+import sys
+import os.path
+import pysvn
+
+if False:
+    from typing import Any, Dict, List, Optional, Text, Tuple, Union
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_svn_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipSVN/" + VERSION)  # type: zulip.Client
+svn = pysvn.Client()  # type: pysvn.Client
+
+path, rev = sys.argv[1:]  # type: Tuple[Text, Text]
+
+# since its a local path, prepend "file://"
+path = "file://" + path
+
+entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0]  # type: Dict[Text, Union[Text, pysvn.Revision, List[Dict[Text, pysvn.Revision]]]]
+message = "**{0}** committed revision r{1} to `{2}`.\n\n> {3}".format(
+    entry['author'],
+    rev,
+    path.split('/')[-1],
+    entry['revprops']['svn:log'])  # type: Text
+
+destination = config.commit_notice_destination(path, rev)  # type: Optional[Dict[Text, Text]]
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}  # type: Dict[str, Any]
+client.send_message(message_data)
--- a/api/integrations/svn/zulip_svn_config.py
+++ b/api/integrations/svn/zulip_svn_config.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "svn-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the svn repository on the server
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for the "evil-master-plan"
+# and "my-super-secret-repository" repos to
+# * stream "commits"
+# * topic "branch_name"
+def commit_notice_destination(path, commit):
+    repo = path.split('/')[-1]
+    if repo not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (repo,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://zulip.example.com"
--- a/api/integrations/trac/zulip_trac.py
+++ b/api/integrations/trac/zulip_trac.py
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Zulip trac plugin -- sends zulips when tickets change.
+#
+# Install by copying this file and zulip_trac_config.py to the trac
+# plugins/ subdirectory, customizing the constants in
+# zulip_trac_config.py, and then adding "zulip_trac" to the
+# components section of the conf/trac.ini file, like so:
+#
+# [components]
+# zulip_trac = enabled
+#
+# You may then need to restart trac (or restart Apache) for the bot
+# (or changes to the bot) to actually be loaded by trac.
+
+from trac.core import Component, implements
+from trac.ticket import ITicketChangeListener
+import sys
+import os.path
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_trac_config as config
+VERSION = "0.9"
+
+if False:
+    from typing import Any
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipTrac/" + VERSION)
+
+def markdown_ticket_url(ticket, heading="ticket"):
+    # type: (Any, str) -> str
+    return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
+
+def markdown_block(desc):
+    # type: (str) -> str
+    return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
+
+def truncate(string, length):
+    # type: (str, int) -> str
+    if len(string) <= length:
+        return string
+    return string[:length - 3] + "..."
+
+def trac_subject(ticket):
+    # type: (Any) -> str
+    return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
+
+def send_update(ticket, content):
+    # type: (Any, str) -> None
+    client.send_message({
+        "type": "stream",
+        "to": config.STREAM_FOR_NOTIFICATIONS,
+        "content": content,
+        "subject": trac_subject(ticket)
+    })
+
+class ZulipPlugin(Component):
+    implements(ITicketChangeListener)
+
+    def ticket_created(self, ticket):
+        # type: (Any) -> None
+        """Called when a ticket is created."""
+        content = "%s created %s in component **%s**, priority **%s**:\n" % \
+            (ticket.values.get("reporter"), markdown_ticket_url(ticket),
+             ticket.values.get("component"), ticket.values.get("priority"))
+        # Include the full subject if it will be truncated
+        if len(ticket.values.get("summary")) > 60:
+            content += "**%s**\n" % (ticket.values.get("summary"),)
+        if ticket.values.get("description") != "":
+            content += "%s" % (markdown_block(ticket.values.get("description")),)
+        send_update(ticket, content)
+
+    def ticket_changed(self, ticket, comment, author, old_values):
+        # type: (Any, str, str, Dict[str, Any]) -> None
+        """Called when a ticket is modified.
+
+        `old_values` is a dictionary containing the previous values of the
+        fields that have changed.
+        """
+        if not (set(old_values.keys()).intersection(set(config.TRAC_NOTIFY_FIELDS)) or
+                (comment and "comment" in set(config.TRAC_NOTIFY_FIELDS))):
+            return
+
+        content = "%s updated %s" % (author, markdown_ticket_url(ticket))
+        if comment:
+            content += ' with comment: %s\n\n' % (markdown_block(comment),)
+        else:
+            content += ":\n\n"
+        field_changes = []
+        for key in old_values.keys():
+            if key == "description":
+                content += '- Changed %s from %s\n\nto %s' % (key, markdown_block(old_values.get(key)),
+                                                              markdown_block(ticket.values.get(key)))
+            elif old_values.get(key) == "":
+                field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
+            elif ticket.values.get(key) == "":
+                field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
+            else:
+                field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
+                                                               ticket.values.get(key)))
+        content += ", ".join(field_changes)
+
+        send_update(ticket, content)
+
+    def ticket_deleted(self, ticket):
+        # type: (Any) -> None
+        """Called when a ticket is deleted."""
+        content = "%s was deleted." % markdown_ticket_url(ticket, heading="Ticket")
+        send_update(ticket, content)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tim Abbott	76809b87a6	Release Zulip Server 1.5.2.	2017-06-01 14:24:51 -07:00
Tim Abbott	5baeb35ac8	invite: Fix invite_by_admins_only to be enforced in backend. Apparently, this setting never actually was wired up to anything other than hiding the UI widget. Huge thanks to Ibram Marzouk from the HackerOne community for finding this security bug.	2017-05-18 11:10:50 -07:00
Tim Abbott	75fbce0532	deps: Upgrade handlebars to latest version. Fixes #3939.	2017-05-18 11:07:33 -07:00
Rafid Aslam	8ad7e08375	deps: Upgrade and move `handlebars` from `static/third` to `npm`. - Remove `handlebars.runtime.js` from static/third and fetch it from npm - Upgrade `handlebars` to 3.0.3. I change the test since there is a patch about line, written in handlebars' v2.0.0-beta.1 release note: "Lines containing only block statements and whitespace are now removed." Fixes part of #1709.	2017-05-18 11:05:42 -07:00
Tim Abbott	bd01b1e2e4	Release Zulip Server 1.5.1.	2017-02-07 11:24:57 -08:00
Tim Abbott	58a7f6085f	stats: Include Zulip and realm name in heading.	2017-02-07 11:24:22 -08:00
Tim Abbott	3367593b52	analytics: Make page-content full height.	2017-02-07 11:24:22 -08:00
Rishi Gupta	1a92ec5d86	analytics: Update bar colors on message_sent_over_time.	2017-02-07 11:24:22 -08:00
Rishi Gupta	7a8d685a71	analytics: Remove portico header and footer from stats.html.	2017-02-07 11:24:22 -08:00
Tim Abbott	3c3a8747c3	upgrade: Stop trying to copy node_modules out of tarballs. Now that we no longer use node_modules at all in production (it's only used to generate static assets), we don't include `node_modules` in the production tarballs, and thus we shouldn't attempt to copy `node_modules` out of the production tarballs when installing. Fixes a regression introduced in `d71f2e7b9b`.	2017-02-07 10:36:38 -08:00