Release Zulip Server 1.5.1.

Now that we no longer use node_modules at all in production (it's only
used to generate static assets), we don't include `node_modules` in
the production tarballs, and thus we shouldn't attempt to copy
`node_modules` out of the production tarballs when installing.

Fixes a regression introduced in
d71f2e7b9b.

.circleci/config.yml

@@ -1,205 +0,0 @@
-# See CircleCI upstream's docs on this config format:
-#   https://circleci.com/docs/2.0/language-python/
-#
-version: 2
-jobs:
-  "trusty-python-3.4":
-    docker:
-      # This is built from tools/circleci/images/trusty/Dockerfile .
-      - image: gregprice/circleci:trusty-python-5.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - run:
-          name: create cache directories
-          command: |
-              dirs=(/srv/zulip-{npm,venv}-cache)
-              sudo mkdir -p "${dirs[@]}"
-              sudo chown -R circleci "${dirs[@]}"
-
-      - restore_cache:
-          keys:
-          - v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          keys:
-          - v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-      - run:
-          name: install dependencies
-          command: |
-            # Install moreutils so we can use `ts` and `mispipe` in the following.
-            sudo apt-get install -y moreutils
-
-            # CircleCI sets the following in Git config at clone time:
-            #   url.ssh://git@github.com.insteadOf https://github.com
-            # This breaks the Git clones in the NVM `install.sh` we run
-            # in `install-node`.
-            # TODO: figure out why that breaks, and whether we want it.
-            #   (Is it an optimization?)
-            rm -f /home/circleci/.gitconfig
-
-            # This is the main setup job for the test suite
-            mispipe "tools/travis/setup-backend" ts
-
-            # Cleaning caches is mostly unnecessary in Circle, because
-            # most builds don't get to write to the cache.
-            # mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
-
-      - save_cache:
-          paths:
-            - /srv/zulip-npm-cache
-          key: v1-npm-base.trusty-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - save_cache:
-          paths:
-            - /srv/zulip-venv-cache
-          key: v1-venv-base.trusty-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-      # TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
-
-      # The moment of truth!  Run the tests.
-
-      - run:
-          name: run backend tests
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            mispipe ./tools/travis/backend ts
-
-      - run:
-          name: run frontend tests
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            mispipe ./tools/travis/frontend ts
-
-      -  run:
-          name: upload coverage report
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            pip install codecov && codecov \
-              || echo "Error in uploading coverage reports to codecov.io."
-
-      # - store_artifacts:  # TODO
-      #     path: var/casper/
-      #     # also /tmp/zulip-test-event-log/
-      #     destination: test-reports
-
-  "xenial-python-3.5":
-    docker:
-      # This is built from tools/circleci/images/xenial/Dockerfile .
-      - image: gregprice/circleci:xenial-python-4.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - run:
-          name: create cache directories
-          command: |
-              dirs=(/srv/zulip-{npm,venv}-cache)
-              sudo mkdir -p "${dirs[@]}"
-              sudo chown -R circleci "${dirs[@]}"
-
-      - restore_cache:
-          keys:
-          - v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          keys:
-          - v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-      - run:
-          name: install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y moreutils
-            rm -f /home/circleci/.gitconfig
-            mispipe "tools/travis/setup-backend" ts
-
-      - save_cache:
-          paths:
-            - /srv/zulip-npm-cache
-          key: v1-npm-base.xenial-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - save_cache:
-          paths:
-            - /srv/zulip-venv-cache
-          key: v1-venv-base.xenial-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-      - run:
-          name: run backend tests
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            mispipe ./tools/travis/backend ts
-
-      -  run:
-          name: upload coverage report
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            pip install codecov && codecov \
-              || echo "Error in uploading coverage reports to codecov.io."
-
-  "bionic-python-3.6":
-    docker:
-      # This is built from tools/circleci/images/bionic/Dockerfile .
-      - image: gregprice/circleci:bionic-python-1.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - run:
-          name: create cache directories
-          command: |
-              dirs=(/srv/zulip-{npm,venv}-cache)
-              sudo mkdir -p "${dirs[@]}"
-              sudo chown -R circleci "${dirs[@]}"
-              # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
-              # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
-              redis-server --daemonize yes
-
-      - restore_cache:
-          keys:
-          - v1-npm-base.bionic-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - restore_cache:
-          keys:
-          - v1-venv-base.bionic-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-      - run:
-          name: install dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y moreutils
-            rm -f /home/circleci/.gitconfig
-            mispipe "tools/travis/setup-backend" ts
-
-      - save_cache:
-          paths:
-            - /srv/zulip-npm-cache
-          key: v1-npm-base.bionic-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-      - save_cache:
-          paths:
-            - /srv/zulip-venv-cache
-          key: v1-venv-base.bionic-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-      - run:
-          name: run backend tests
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            mispipe ./tools/travis/backend ts
-
-      -  run:
-          name: upload coverage report
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            pip install codecov && codecov \
-              || echo "Error in uploading coverage reports to codecov.io."
-
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - "trusty-python-3.4"
-      - "xenial-python-3.5"
-      - "bionic-python-3.6"

.codecov.yml

@@ -1,10 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        threshold: 0.50
-        base: auto
-    patch: off

.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG

.editorconfig

@@ -6,16 +6,10 @@ charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 
-[*.{sh,py,pyi,js,json,yml,xml,css,md,markdown,handlebars,html}]
+[*.{sh,py,js, json,yml,xml, css, md,markdown, handlebars,html}]
 indent_style = space
 indent_size = 4
 
-[*.{py}]
-max_line_length = 110
-
-[*.{js}]
-max_line_length = 120
-
 [*.{svg,rb,pp,pl}]
 indent_style = space
 indent_size = 2

.eslintignore

@@ -1,2 +1,3 @@
+static/js/bundle.js
 static/js/blueslip.js
-static/webpack-bundles
+puppet/zulip_ops/files/statsd/local.js

.eslintrc.json

@@ -1,385 +1,245 @@
 {
     "env": {
-        "node": true,
-        "es6": true
-    },
-    "parserOptions": {
-        "sourceType": "module"
+        "node": true
     },
     "globals": {
         "$": false,
-        "ClipboardJS": false,
-        "Dict": false,
-        "FetchStatus": false,
-        "Filter": false,
-        "Handlebars": false,
-        "LightboxCanvas": false,
-        "MessageListData": false,
-        "MessageListView": false,
-        "PerfectScrollbar": false,
-        "Plotly": false,
-        "SockJS": false,
-        "Socket": false,
-        "Sortable": false,
-        "WinChan": false,
-        "XDate": false,
         "_": false,
-        "activity": false,
-        "admin": false,
-        "admin_sections": false,
-        "alert_words": false,
-        "alert_words_ui": false,
-        "attachments_ui": false,
-        "avatar": false,
-        "blueslip": false,
-        "bot_data": false,
-        "bridge": false,
-        "buddy_data": false,
-        "buddy_list": false,
-        "channel": false,
-        "click_handlers": false,
-        "colorspace": false,
-        "common": false,
-        "components": false,
-        "compose": false,
-        "compose_actions": false,
-        "compose_fade": false,
-        "compose_pm_pill": false,
-        "compose_state": false,
-        "compose_ui": false,
-        "composebox_typeahead": false,
-        "condense": false,
-        "copy_and_paste": false,
-        "csrf_token": false,
-        "current_msg_list": true,
-        "drafts": false,
-        "echo": false,
-        "emoji": false,
-        "emoji_codes": false,
-        "emoji_picker": false,
-        "favicon": false,
-        "feature_flags": false,
-        "fenced_code": false,
-        "flatpickr": false,
-        "floating_recipient_bar": false,
-        "gear_menu": false,
-        "hash_util": false,
-        "hashchange": false,
-        "home_msg_list": false,
-        "hotspots": false,
-        "i18n": false,
-        "info_overlay": false,
-        "input_pill": false,
-        "invite": false,
         "jQuery": false,
-        "katex": false,
-        "keydown_util": false,
-        "lightbox": false,
-        "list_cursor": false,
-        "list_render": false,
-        "list_util": false,
-        "loading": false,
-        "localStorage": false,
-        "local_message": false,
-        "localstorage": false,
-        "markdown": false,
+        "Spinner": false,
+        "Handlebars": false,
+        "XDate": false,
+        "zxcvbn": false,
+        "LazyLoad": false,
+        "Dropbox": false,
+        "SockJS": false,
         "marked": false,
-        "md5": false,
-        "message_edit": false,
-        "message_events": false,
-        "message_fetch": false,
-        "message_flags": false,
-        "message_list": false,
-        "message_live_update": false,
-        "message_scroll": false,
-        "message_store": false,
-        "message_util": false,
-        "message_viewport": false,
-        "moment": false,
-        "muting": false,
-        "muting_ui": false,
-        "narrow": false,
-        "narrow_state": false,
-        "navigate": false,
-        "night_mode": false,
-        "notifications": false,
-        "overlays": false,
-        "padded_widget": false,
+        "i18n": false,
+        "bridge": false,
         "page_params": false,
-        "panels": false,
-        "people": false,
-        "pm_conversations": false,
-        "pm_list": false,
-        "pointer": false,
-        "popovers": false,
-        "presence": false,
-        "pygments_data": false,
-        "reactions": false,
-        "realm_icon": false,
-        "recent_senders": false,
-        "reload": false,
-        "reload_state": false,
-        "reminder": false,
-        "resize": false,
-        "rows": false,
-        "rtl": false,
-        "run_test": false,
-        "schema": false,
-        "scroll_bar": false,
-        "scroll_util": false,
-        "search": false,
-        "search_pill": false,
-        "search_pill_widget": false,
-        "search_suggestion": false,
-        "search_util": false,
-        "sent_messages": false,
-        "server_events": false,
-        "server_events_dispatch": false,
-        "settings": false,
-        "settings_account": false,
-        "settings_bots": false,
-        "settings_display": false,
-        "settings_emoji": false,
-        "settings_filters": false,
-        "settings_invites": false,
-        "settings_muting": false,
-        "settings_notifications": false,
-        "settings_org": false,
-        "settings_panel_menu": false,
-        "settings_profile_fields": false,
-        "settings_sections": false,
-        "settings_streams": false,
-        "settings_toggle": false,
-        "settings_ui": false,
-        "settings_user_groups": false,
-        "settings_users": false,
-        "starred_messages": false,
-        "stream_color": false,
-        "stream_create": false,
-        "stream_data": false,
-        "stream_edit": false,
-        "stream_events": false,
-        "stream_list": false,
-        "stream_muting": false,
-        "stream_popover": false,
-        "stream_sort": false,
-        "submessage": false,
-        "subs": false,
-        "tab_bar": false,
-        "templates": false,
-        "tictactoe_widget": false,
-        "timerender": false,
-        "toMarkdown": false,
-        "todo_widget": false,
-        "top_left_corner": false,
-        "topic_data": false,
-        "topic_generator": false,
-        "topic_list": false,
-        "transmit": false,
-        "tutorial": false,
+        "status_classes": false,
+        "password_quality": false,
+        "csrf_token": false,
         "typeahead_helper": false,
-        "typing": false,
-        "typing_data": false,
-        "typing_events": false,
-        "typing_status": false,
+        "popovers": false,
+        "server_events": false,
         "ui": false,
-        "ui_report": false,
-        "ui_util": false,
-        "unread": false,
-        "unread_ops": false,
-        "unread_ui": false,
-        "upload": false,
-        "upload_widget": false,
-        "user_events": false,
-        "user_groups": false,
-        "user_pill": false,
-        "user_search": false,
+        "stream_color": false,
+        "people": false,
+        "navigate": false,
+        "settings": false,
+        "resize": false,
+        "loading": false,
+        "compose": false,
+        "compose_fade": false,
+        "subs": false,
+        "timerender": false,
+        "message_live_update": false,
+        "message_edit": false,
+        "reload": false,
+        "composebox_typeahead": false,
+        "search": false,
+        "topic_list": false,
+        "gear_menu": false,
+        "hashchange": false,
+        "message_list": false,
+        "Filter": false,
+        "pointer": false,
         "util": false,
-        "voting_widget": false,
-        "widgetize": false,
-        "zcommand": false,
-        "zform": false,
-        "zxcvbn": false
+        "MessageListView": false,
+        "blueslip": false,
+        "rows": false,
+        "WinChan": false,
+        "muting_ui": false,
+        "Socket": false,
+        "channel": false,
+        "components": false,
+        "viewport": false,
+        "avatar": false,
+        "feature_flags": false,
+        "search_suggestion": false,
+        "referral": false,
+        "notifications": false,
+        "message_flags": false,
+        "bot_data": false,
+        "stream_list": false,
+        "narrow": false,
+        "admin": false,
+        "stream_data": false,
+        "muting": false,
+        "Dict": false,
+        "unread": false,
+        "alert_words_ui": false,
+        "message_store": false,
+        "favicon": false,
+        "condense": false,
+        "floating_recipient_bar": false,
+        "tab_bar": false,
+        "emoji": false,
+        "activity": false,
+        "invite": false,
+        "colorspace": false,
+        "reactions": false,
+        "tutorial": false,
+        "templates": false,
+        "alert_words": false,
+        "fenced_code": false,
+        "echo": false,
+        "localstorage": false,
+        "current_msg_list": true,
+        "home_msg_list": false,
+        "pm_list": false,
+        "unread_ui": false,
+        "user_events": false,
+        "Plotly": false,
+        "emoji_codes": false
     },
-    "plugins": [
-        "eslint-plugin-empty-returns"
-    ],
     "rules": {
-        "array-callback-return": "error",
-        "array-bracket-spacing": "error",
-        "arrow-spacing": [ "error", { "before": true, "after": true } ],
-        "block-scoped-var": 2,
-        "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
-        "camelcase": 0,
-        "comma-dangle": [ "error",
-            {
-                "arrays": "always-multiline",
-                "objects": "always-multiline",
-                "imports": "always-multiline",
-                "exports": "always-multiline",
-                "functions": "never"
-            }
-        ],
-        "complexity": [ 0, 4 ],
-        "curly": 2,
-        "dot-notation": [ "error", { "allowKeywords": true } ],
-        "empty-returns/main": "error",
-        "eol-last": [ "error", "always" ],
-        "eqeqeq": 2,
-        "func-style": [ "off", "expression" ],
-        "guard-for-in": 2,
-        "indent": ["error", 4, {
-            "ArrayExpression": "first",
-            "outerIIFEBody": 0,
-            "ObjectExpression": "first",
-            "SwitchCase": 0,
-            "CallExpression": {"arguments": "first"},
-            "FunctionExpression": {"parameters": "first"},
-            "FunctionDeclaration": {"parameters": "first"}
-        }],
-        "keyword-spacing": [ "error",
-            {
-                "before": true,
-                "after": true,
-                "overrides": {
-                    "return": { "after": true },
-                    "throw": { "after": true },
-                    "case": { "after": true }
-                }
-            }
-        ],
-        "max-depth": [ 0, 4 ],
-        "max-len": [ "error", 100, 2,
-            {
-                "ignoreUrls": true,
-                "ignoreComments": false,
-                "ignoreRegExpLiterals": true,
-                "ignoreStrings": true,
-                "ignoreTemplateLiterals": true
-            }
-        ],
-        "max-params": [ 0, 3 ],
-        "max-statements": [ 0, 10 ],
-        "new-cap": [ "error",
-            {
-                "newIsCap": true,
-                "capIsNew": false
-            }
-        ],
-        "new-parens": 2,
+        "no-restricted-syntax": 0,
+        "no-nested-ternary": 0,
+        "spaced-comment": 0,
+        "space-infix-ops": 0,
         "newline-per-chained-call": 0,
-        "no-alert": 2,
-        "no-array-constructor": "error",
-        "no-bitwise": 2,
-        "no-caller": 2,
+        "no-whitespace-before-property": 0,
+        "padded-blocks": 0,
+        "space-in-parens": 0,
+        "eol-last": ["error", "always"],
+        "no-unneeded-ternary": ["error", { "defaultAssignment": false }],
         "no-case-declarations": "error",
-        "no-catch-shadow": 2,
-        "no-console": 0,
+        "eqeqeq": ["error", "allow-null"],
+        "no-duplicate-imports": "error",
+        "no-undef": "error",
+        "dot-notation": ["error", { "allowKeywords": true }],
+        "no-iterator": "error",
+        "no-dupe-class-members": "error",
+        "no-useless-constructor": "error",
+        "prefer-const": ["error", {
+            "destructuring": "any",
+            "ignoreReadBeforeAssign": true
+        }],
         "no-const-assign": "error",
+        "no-new-object": 2,
+        "quote-props": ["error", "as-needed", {
+            "keywords": false,
+            "unnecessary": true,
+            "numbers": false
+        }],
+        "no-array-constructor": "error",
+        "array-callback-return": "error",
+        "template-curly-spacing": "error",
+        //The Zulip codebase complies partially with the "no-useless-escape" rule; only regex expressions haven't been updated yet.
+        //Updated regex expressions are currently being tested in casper files and will decide about a potential future enforcement of this rule.
+        "no-useless-escape": 0,
+        "func-style": ["off", "expression"],
+        "wrap-iife": ["error", "outside", { "functionPrototypeMethods": false }],
+        "no-new-func": "error",
+        "space-before-function-paren": ["error", { "anonymous": "always", "named": "never", "asyncArrow": "always" }],
+        "no-param-reassign": 0,
+        "prefer-spread": "error",
+        "arrow-spacing": ["error", { "before": true, "after": true }],
+        "no-alert": 2,
+        "no-array-constructor": 2,
+        "no-caller": 2,
+        "no-bitwise": 2,
+        "no-catch-shadow": 2,
+        "comma-dangle": ["error", {
+            "arrays": "always-multiline",
+            "objects": "always-multiline",
+            "imports": "always-multiline",
+            "exports": "always-multiline",
+            "functions": "never"
+        }],
+        "no-console": 0,
         "no-control-regex": 2,
         "no-debugger": 2,
-        "no-delete-var": 2,
         "no-div-regex": 2,
-        "no-dupe-class-members": "error",
         "no-dupe-keys": 2,
-        "no-duplicate-imports": "error",
         "no-else-return": 2,
         "no-empty": 2,
         "no-empty-character-class": 2,
         "no-eq-null": 2,
         "no-eval": 2,
         "no-ex-assign": 2,
-        "no-extra-parens": ["error", "all"],
         "no-extra-semi": 2,
-        "no-fallthrough": 2,
-        "no-floating-decimal": 2,
         "no-func-assign": 2,
+        "no-floating-decimal": 2,
         "no-implied-eval": 2,
-        "no-iterator": "error",
-        "no-label-var": 2,
-        "no-labels": 2,
-        "no-loop-func": 2,
-        "no-mixed-requires": [ 0, false ],
-        "no-multi-str": 2,
-        "no-native-reassign": 2,
-        "no-nested-ternary": 0,
-        "no-new-func": "error",
-        "no-new-object": 2,
-        "no-new-wrappers": 2,
-        "no-obj-calls": 2,
+        "no-with": 2,
+        "no-fallthrough": 2,
+        "no-unreachable": 2,
+        "no-undef": 2,
+        "no-undef-init": 2,
+        "no-unused-expressions": 2,
         "no-octal": 2,
         "no-octal-escape": 2,
-        "no-param-reassign": 0,
+        "no-obj-calls": 2,
+        "no-multi-str": 2,
+        "no-new-wrappers": 2,
+        "no-new": 2,
+        "no-new-func": 2,
+        "no-native-reassign": 2,
         "no-plusplus": 2,
+        "no-delete-var": 2,
+        "no-return-assign": 2,
+        "no-new-object": 2,
+        "no-label-var": 2,
+        "no-ternary": 0,
+        "no-self-compare": 2,
+        "no-sync": 2,
+        "no-underscore-dangle": 0,
+        "no-loop-func": 2,
+        "no-labels": 2,
+        "no-unused-vars": ["error", { "vars": "local", "args": "after-used",
+                                      "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
+                                    }],
+        "no-script-url": 2,
         "no-proto": 2,
+        "no-iterator": 2,
+        "no-mixed-requires": [0, false],
+        "no-extra-parens": ["error", "functions"],
+        "no-shadow": 0,
+        "no-use-before-define": 2,
         "no-redeclare": 2,
         "no-regex-spaces": 2,
-        "no-restricted-syntax": 0,
-        "no-return-assign": 2,
-        "no-script-url": 2,
-        "no-self-compare": 2,
-        "no-shadow": 0,
-        "no-sync": 2,
-        "no-ternary": 0,
-        "no-trailing-spaces": 2,
-        "no-undef": "error",
-        "no-undef-init": 2,
-        "no-underscore-dangle": 0,
-        "no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
-        "no-unreachable": 2,
-        "no-unused-expressions": 2,
-        "no-unused-vars": [ "error",
-            {
-                "vars": "local",
-                "args": "after-used",
-                "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
-            }
-        ],
-        "no-use-before-define": 2,
-        "no-useless-constructor": "error",
-        // The Zulip codebase complies partially with the "no-useless-escape"
-        // rule; only regex expressions haven't been updated yet.
-        // Updated regex expressions are currently being tested in casper
-        // files and will decide about a potential future enforcement of this rule.
-        "no-useless-escape": 0,
-        "space-unary-ops": 2,
-        "no-whitespace-before-property": 2,
-        "no-with": 2,
-        "one-var": [ "error", "never" ],
-        "padded-blocks": 0,
-        "prefer-const": [ "error",
-            {
-                "destructuring": "any",
-                "ignoreReadBeforeAssign": true
-            }
-        ],
-        "quote-props": [ "error", "as-needed",
-            {
-                "keywords": false,
-                "unnecessary": true,
-                "numbers": false
-            }
-        ],
-        "quotes": [ 0, "single" ],
+        "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
+        "block-scoped-var": 2,
+        "camelcase": 0,
+        "complexity": [0, 4],
+        "curly": 2,
+        "dot-notation": 2,
+        "guard-for-in": 2,
+        "max-depth": [0, 4],
+        "max-len": ["error", 100, 2, {
+            "ignoreUrls": true,
+            "ignoreComments": false,
+            "ignoreRegExpLiterals": true,
+            "ignoreStrings": true,
+            "ignoreTemplateLiterals": true
+        }],
+        "max-params": [0, 3],
+        "max-statements": [0, 10],
+        "new-cap": ["error", { "newIsCap": true, "capIsNew": false }],
+        "new-parens": 2,
+        "one-var": ["error", "never"],
+        "quotes": [0, "single"],
+        "quote-props": ["error", "as-needed", { "keywords": false, "unnecessary": true, "numbers": false }],
         "radix": 2,
         "semi": 2,
-        "space-before-blocks": 2,
-        "space-before-function-paren": [ "error",
-            {
-                "anonymous": "always",
-                "named": "never",
-                "asyncArrow": "always"
+        "keyword-spacing": ["error", {
+            "before": true,
+            "after": true,
+            "overrides": {
+                "return": { "after": true },
+                "throw": { "after": true },
+                "case": { "after": true }
             }
-        ],
-        "space-in-parens": 2,
-        "space-infix-ops": 2,
-        "spaced-comment": 0,
+        }],
+        "space-before-blocks": 2,
         "strict": 0,
-        "template-curly-spacing": "error",
         "unnecessary-strict": 0,
         "use-isnan": 2,
-        "valid-typeof": [ "error", { "requireStringLiterals": true } ],
-        "wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
+        "valid-typeof": ["error", { "requireStringLiterals": true }],
+        "wrap-iife": 2,
         "wrap-regex": 0,
         "yoda": 2
     }

.gitattributes

@@ -1,14 +1,29 @@
 *   text=auto eol=lf
 *.gif binary
 *.jpg binary
-*.jpeg binary
 *.eot binary
 *.woff binary
-*.woff2 binary
 *.svg binary
 *.ttf binary
 *.png binary
 *.otf binary
 *.tif binary
-*.ogg binary
-yarn.lock binary
+.gitignore export-ignore
+.gitattributes export-ignore
+/static/assets export-ignore
+/bots export-ignore
+/corporate export-ignore
+/static export-ignore
+/tools export-ignore
+/zilencer export-ignore
+/templates/corporate export-ignore
+/templates/zilencer export-ignore
+/puppet/zulip_internal export-ignore
+/zproject/local_settings.py export-ignore
+/zproject/test_settings.py export-ignore
+/zerver/fixtures export-ignore
+/zerver/tests export-ignore
+/frontend_tests export-ignore
+/node_modules export-ignore
+/humbug export-ignore
+/locale export-ignore

.github/pull_request_template.md

@@ -1,14 +0,0 @@
-<!-- What's this PR for?  (Just a link to an issue is fine.) -->
-
-
-**Testing Plan:** <!-- How have you tested? -->
-
-
-**GIFs or Screenshots:** <!-- If a UI change.  See:
-  https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
-  -->
-
-
-<!-- Also be sure to make clear, coherent commits:
-  https://zulip.readthedocs.io/en/latest/contributing/version-control.html
-  -->

.gitignore

@@ -1,75 +1,29 @@
-# Quick format and style primer:
-#
-#  * If a pattern is meant only for a specific location, it should have a
-#    leading slash, like `/staticfiles.json`.
-#    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
-#      will do, but this makes a confusing pattern.  Adding a leading slash
-#      is clearer.
-#
-#  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
-#    match in any subdirectory.
-#
-#  * Subdirectories with several internal things to ignore get their own
-#    `.gitignore` files.
-#
-#  * Comments must be on their own line.  (Otherwise they don't work.)
-#
-# See `git help ignore` for details on the format.
-
-## Config files for the dev environment
-/zproject/dev-secrets.conf
-/tools/conf.ini
-/tools/custom_provision
-/tools/droplets/conf.ini
-
-## Byproducts of setting up and using the dev environment
 *.pyc
-package-lock.json
-
-/.vagrant
-/var
-
-/.dmypy.json
-
-# Dockerfiles generated for CircleCI
-/tools/circleci/images
-
-# Static build
-*.mo
-npm-debug.log
-/node_modules
-/prod-static
-/staticfiles.json
-/webpack-stats-production.json
-/yarn-error.log
-
-# Test / analysis tools
-.coverage
-
-## Files (or really symlinks) created in a prod deployment
-/zproject/prod_settings.py
-/zulip-current-venv
-/zulip-py3-venv
-
-## Files left by various editors and local environments
-# (Ideally these should be in everyone's respective personal gitignore files.)
 *~
+/prod-static
 *.sw[po]
-.idea
+*.DS_Store
 .kdev4
+.idea
 zulip.kdev4
+coverage/
+.coverage
+/queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
+.vagrant
+/zproject/dev-secrets.conf
+static/js/bundle.js
+static/generated/emoji
+static/generated/github-contributors.json
+static/locale/language_options.json
+/node_modules
+/staticfiles.json
+npm-debug.log
+*.mo
+var/*
 .vscode/
-*.DS_Store
-# .cache/ is generated by VSCode's test runner
-.cache/
-.eslintcache
-
-## Miscellaneous
-# (Ideally this section is empty.)
-zthumbor/thumbor_local_settings.py
-.transifexrc
+tools/conf.ini

.gitlint

@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing, title-imperative-mood
-
-extra-path=tools/lib/gitlint-rules.py
-
-[title-match-regex-allow-exception]
-regex=^(.+:\ )?[A-Z].+\.$
-
-[title-max-length]
-line-length=76
-
-[body-max-line-length]
-line-length=76

.isort.cfg

@@ -1,10 +0,0 @@
-[settings]
-line_length = 79
-multi_line_output = 2
-balanced_wrapping = true
-known_third_party = django, ujson, sqlalchemy
-known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
-sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
-lines_after_imports = 1
-# See the comment related to ioloop_logging for why this is skipped.
-skip = zerver/management/commands/runtornado.py

.stylelintrc

@@ -1,63 +0,0 @@
-{
-    "rules": {
-        # Stylistic rules for CSS.
-        "function-comma-space-after": "always",
-        "function-comma-space-before": "never",
-        "function-max-empty-lines": 0,
-        "function-whitespace-after": "always",
-
-        "value-keyword-case": "lower",
-        "value-list-comma-newline-after": "always-multi-line",
-        "value-list-comma-space-after": "always-single-line",
-        "value-list-comma-space-before": "never",
-        "value-list-max-empty-lines": 0,
-
-        "unit-case": "lower",
-        "property-case": "lower",
-        "color-hex-case": "lower",
-
-        "declaration-bang-space-before": "always",
-        "declaration-colon-newline-after": "always-multi-line",
-        "declaration-colon-space-after": "always-single-line",
-        "declaration-colon-space-before": "never",
-        "declaration-block-semicolon-newline-after": "always",
-        "declaration-block-semicolon-space-before": "never",
-        "declaration-block-trailing-semicolon": "always",
-
-        "block-closing-brace-empty-line-before": "never",
-        "block-closing-brace-newline-after": "always",
-        "block-closing-brace-newline-before": "always",
-        "block-opening-brace-newline-after": "always",
-        "block-opening-brace-space-before": "always",
-
-        "selector-attribute-brackets-space-inside": "never",
-        "selector-attribute-operator-space-after": "never",
-        "selector-attribute-operator-space-before": "never",
-        "selector-combinator-space-after": "always",
-        "selector-combinator-space-before": "always",
-        "selector-descendant-combinator-no-non-space": true,
-        "selector-pseudo-class-parentheses-space-inside": "never",
-        "selector-pseudo-element-case": "lower",
-        "selector-pseudo-element-colon-notation": "double",
-        "selector-type-case": "lower",
-        "selector-list-comma-newline-after": "always",
-        "selector-list-comma-space-before": "never",
-
-        "media-feature-colon-space-after": "always",
-        "media-feature-colon-space-before": "never",
-        "media-feature-name-case": "lower",
-        "media-feature-parentheses-space-inside": "never",
-        "media-feature-range-operator-space-after": "always",
-        "media-feature-range-operator-space-before": "always",
-        "media-query-list-comma-newline-after": "always",
-        "media-query-list-comma-space-before": "never",
-
-        "at-rule-name-case": "lower",
-        "at-rule-name-space-after": "always",
-        "at-rule-semicolon-newline-after": "always",
-        "at-rule-semicolon-space-before": "never",
-
-        "comment-whitespace-inside": "always",
-        "indentation": 4,
-    }
-}

.travis.yml

@@ -1,67 +1,54 @@
-# See https://zulip.readthedocs.io/en/latest/testing/travis.html for
-# high-level documentation on our Travis CI setup.
 dist: trusty
-group: deprecated-2017Q4
+before_install:
+   - nvm install 0.10
 install:
-  # Disable sometimes-broken sources.list in Travis base images
-  - sudo rm -vf /etc/apt/sources.list.d/*
-  - sudo apt-get update
-
   # Disable Travis CI's built-in NVM installation
-  - mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
-
-  # Install codecov, the library for the code coverage reporting tool we use
-  # With a retry to minimize impact of transient networking errors.
-  - mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
-
-  # This is the main setup job for the test suite
-  - mispipe "tools/travis/setup-$TEST_SUITE" ts
-
-  # Clean any caches that are not in use to avoid our cache
-  # becoming huge.
-  - mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
-
-script:
-  # We unset GEM_PATH here as a hack to work around Travis CI having
-  # broken running their system puppet with Ruby.  See
-  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
-  - unset GEM_PATH
-  - mispipe "./tools/travis/$TEST_SUITE" ts
+  - mv ~/.nvm ~/.travis-nvm-disabled
+  - pip install coveralls
+  - tools/travis/setup-$TEST_SUITE
+  - tools/clean-venv-cache --travis
 cache:
-  yarn: true
-  apt: false
-  directories:
+  - apt: false
+  - directories:
     - $HOME/zulip-venv-cache
-    - $HOME/zulip-npm-cache
-    - $HOME/zulip-emoji-cache
+    - node_modules
     - $HOME/node
-    - $HOME/misc
 env:
   global:
+    - COVERAGE_FILE=var/.coverage
+    - COVERALLS_PARALLEL=true
+    - COVERALLS_SERVICE_NAME=travis-pro
+    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
     - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
 language: python
-# Our test suites generally run on Python 3.4, the version in
-# Ubuntu 14.04 trusty, which is the oldest OS release we support.
+python:
+  - "2.7"
+  - "3.4"
 matrix:
   include:
-    # Travis will actually run the jobs in the order they're listed here;
-    # that doesn't seem to be documented, but it's what we see empirically.
-    # We only get 4 jobs running at a time, so we try to make the first few
-    # the most likely to break.
+    - python: "3.4"
+      env: TEST_SUITE=static-analysis
     - python: "3.4"
       env: TEST_SUITE=production
-    # Other suites moved to CircleCI -- see .circleci/.
+    - python: "2.7"
+      env: TEST_SUITE=production
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
 sudo: required
+services:
+- docker
 addons:
   artifacts:
     paths:
-      # Casper debugging data (screenshots, etc.) is super useful for
-      # debugging test flakes.
       - $(ls var/casper/* | tr "\n" ":")
       - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
   postgresql: "9.3"
-  apt:
-    packages:
-      - moreutils
 after_success:
-  - codecov
+  coveralls
+notifications:
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN

.tx/config

@@ -1,6 +1,5 @@
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
 [zulip.djangopo]
 source_file = static/locale/en/LC_MESSAGES/django.po
@@ -13,21 +12,3 @@ source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
-
-[zulip.mobile]
-source_file = static/locale/en/mobile.json
-source_lang = en
-type = KEYVALUEJSON
-file_filter = static/locale/<lang>/mobile.json
-
-[zulip-test.djangopo]
-source_file = static/locale/en/LC_MESSAGES/django.po
-source_lang = en
-type = PO
-file_filter = static/locale/<lang>/LC_MESSAGES/django.po
-
-[zulip-test.translationsjson]
-source_file = static/locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-file_filter = static/locale/<lang>/translations.json

CODE_OF_CONDUCT.md

@@ -1,105 +0,0 @@
-# Zulip Code of Conduct
-
-Like the technical community as a whole, the Zulip team and community is
-made up of a mixture of professionals and volunteers from all over the
-world, working on every aspect of the mission, including mentorship,
-teaching, and connecting people.
-
-Diversity is one of our huge strengths, but it can also lead to
-communication issues and unhappiness. To that end, we have a few ground
-rules that we ask people to adhere to. This code applies equally to
-founders, mentors, and those seeking help and guidance.
-
-This isn't an exhaustive list of things that you can't do. Rather, take it
-in the spirit in which it's intended --- a guide to make it easier to enrich
-all of us and the technical communities in which we participate.
-
-## Expected Behavior
-
-The following behaviors are expected and requested of all community members:
-
-* Participate. In doing so, you contribute to the health and longevity of
-  the community.
-* Exercise consideration and respect in your speech and actions.
-* Attempt collaboration before conflict. Assume good faith.
-* Refrain from demeaning, discriminatory, or harassing behavior and speech.
-* Take action or alert community leaders if you notice a dangerous
-  situation, someone in distress, or violations of this code, even if they
-  seem inconsequential.
-* Community event venues may be shared with members of the public; be
-  respectful to all patrons of these locations.
-
-## Unacceptable Behavior
-
-The following behaviors are considered harassment and are unacceptable
-within the Zulip community:
-
-* Jokes or derogatory language that singles out members of any race,
-  ethnicity, culture, national origin, color, immigration status, social and
-  economic class, educational level, language proficiency, sex, sexual
-  orientation, gender identity and expression, age, size, family status,
-  political belief, religion, and mental and physical ability.
-* Violence, threats of violence, or violent language directed against
-  another person.
-* Disseminating or threatening to disseminate another person's personal
-  information.
-* Personal insults of any sort.
-* Posting or displaying sexually explicit or violent material.
-* Inappropriate photography or recording.
-* Deliberate intimidation, stalking, or following (online or in person).
-* Unwelcome sexual attention. This includes sexualized comments or jokes,
-  inappropriate touching or groping, and unwelcomed sexual advances.
-* Sustained disruption of community events, including talks and
-  presentations.
-* Advocating for, or encouraging, any of the behaviors above.
-
-## Reporting and Enforcement
-
-Harassment and other code of conduct violations reduce the value of the
-community for everyone. If someone makes you or anyone else feel unsafe or
-unwelcome, please report it to the community organizers at
-zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
-report either personally or anonymously.
-
-If a community member engages in unacceptable behavior, the community
-organizers may take any action they deem appropriate, up to and including a
-temporary ban or permanent expulsion from the community without warning (and
-without refund in the case of a paid event).
-
-If someone outside the development community (e.g. a user of the Zulip
-software) engages in unacceptable behavior that affects someone in the
-community, we still want to know. Even if we don't have direct control over
-the violator, the community organizers can still support the people
-affected, reduce the chance of a similar violation in the future, and take
-any direct action we can.
-
-The nature of reporting means it can only help after the fact. If you see
-something you can do while a violation is happening, do it. A lot of the
-harms of harassment and other violations can be mitigated by the victim
-knowing that the other people present are on their side.
-
-All reports will be kept confidential. In some cases we may determine that a
-public statement will need to be made. In such cases, the identities of all
-victims and reporters will remain confidential unless those individuals
-instruct us otherwise.
-
-## Scope
-
-We expect all community participants (contributors, paid or otherwise,
-sponsors, and other guests) to abide by this Code of Conduct in all
-community venues, online and in-person, as well as in all private
-communications pertaining to community business.
-
-This Code of Conduct and its related procedures also applies to unacceptable
-behavior occurring outside the scope of community activities when such
-behavior has the potential to adversely affect the safety and well-being of
-community members.
-
-## License and Attribution
-
-This Code of Conduct is adapted from the
-[Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
-[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
-under a
-[Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
-license.

CONTRIBUTING.md

@@ -1,341 +0,0 @@
-# Contributing to Zulip
-
-Welcome to the Zulip community!
-
-## Community
-
-The
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-is the primary communication forum for the Zulip community. It is a good
-place to start whether you have a question, are a new contributor, are a new
-user, or anything else. Make sure to read the
-[community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
-before posting. The Zulip community is also governed by a
-[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
-
-You can subscribe to zulip-devel@googlegroups.com for a lower traffic (~1
-email/month) way to hear about things like mentorship opportunities with Google
-Code-in, in-person sprints at conferences, and other opportunities to
-contribute.
-
-## Ways to contribute
-
-To make a code or documentation contribution, read our
-[step-by-step guide](#your-first-codebase-contribution) to getting
-started with the Zulip codebase. A small sample of the type of work that
-needs doing:
-* Bug squashing and feature development on our Python/Django
-  [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip), React Native
-  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
-  [desktop app](https://github.com/zulip/zulip-electron).
-* Building out our
-  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
-* [Writing an integration](https://zulipchat.com/api/integration-guide).
-* Improving our [user](https://zulipchat.com/help/) or
-  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
-* [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
-  and manually testing pull requests.
-
-**Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. We list a few of them below:
-
-* [Reporting issues](#reporting-issues), including both feature requests and
-  bug reports.
-* [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
-* [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip.
-* [Outreach](#zulip-outreach): Star us on GitHub, upvote us
-  on product comparison sites, or write for [the Zulip blog](http://blog.zulip.org/).
-
-## Your first (codebase) contribution
-
-This section has a step by step guide to starting as a Zulip codebase
-contributor. It's long, but don't worry about doing all the steps perfectly;
-no one gets it right the first time, and there are a lot of people available
-to help.
-* First, make an account on the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
-  paying special attention to the community norms. If you'd like, introduce
-  yourself in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
-  your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken as you started using the
-  product.
-* Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-* [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-  getting help in
-  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
-  if you run into any troubles.
-* Read the
-  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
-  and do the Git tutorial (coming soon) if you are unfamiliar with Git,
-  getting help in
-  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if you run
-  into any troubles.
-* Sign the
-  [Dropbox Contributor License Agreement](https://opensource.dropbox.com/cla/).
-
-### Picking an issue
-
-Now, you're ready to pick your first issue! There are hundreds of open issues
-in the main codebase alone. This section will help you find an issue to work
-on.
-
-* If you're interested in
-  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
-  [desktop](https://github.com/zulip/zulip-electron/issues?q=is%3Aopen+is%3Aissue),
-  or
-  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
-  development, check the respective links for open issues, or post in
-  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
-  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
-  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
-* For the main server and web repository, start by looking through issues
-  with the label
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A"good+first+issue").
-  These are smaller projects particularly suitable for a first contribution.
-* We also partition all of our issues in the main repo into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-* If the lists of issues are overwhelming, post in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
-  bit about your background and interests, and we'll help you out. The most
-  important thing to say is whether you're looking for a backend (Python),
-  frontend (JavaScript), mobile (React Native), desktop (Electron),
-  documentation (English) or visual design (JavaScript + CSS) issue, and a
-  bit about your programming experience and available time.
-
-We also welcome suggestions of features that you feel would be valuable or
-changes that you feel would make Zulip a better open source project. If you
-have a new feature you'd like to add, we recommend you start by posting in
-[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
-feature idea and the problem that you're hoping to solve.
-
-Other notes:
-* For a first pull request, it's better to aim for a smaller contribution
-  than a bigger one. Many first contributions have fewer than 10 lines of
-  changes (not counting changes to tests).
-* The full list of issues looking for a contributor can be found with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  and
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels.
-* For most new contributors, there's a lot to learn while making your first
-  pull request. It's OK if it takes you a while; that's normal! You'll be
-  able to work a lot faster as you build experience.
-
-### Working on an issue
-
-To work on an issue, claim it by adding a comment with `@zulipbot claim` to
-the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
-workflow bot; it will assign you to the issue and label the issue as "in
-progress". Some additional notes:
-
-* You can only claim issues with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  or
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels. Zulipbot will give you an error if you try to claim an issue
-  without one of those labels.
-* You're encouraged to ask questions on how to best implement or debug your
-  changes -- the Zulip maintainers are excited to answer questions to help
-  you stay unblocked and working efficiently. You can ask questions on
-  chat.zulip.org, or on the GitHub issue or pull request.
-* We encourage early pull requests for work in progress. Prefix the title of
-  work in progress pull requests with `[WIP]`, and remove the prefix when
-  you think it might be mergeable and want it to be reviewed.
-* After updating a PR, add a comment to the GitHub thread mentioning that it
-  is ready for another review. GitHub only notifies maintainers of the
-  changes when you post a comment, so if you don't, your PR will likely be
-  neglected by accident!
-
-### And beyond
-
-A great place to look for a second issue is to look for issues with the same
-`area:` label as the last issue you resolved. You'll be able to reuse the
-work you did learning how that part of the codebase works. Also, the path to
-becoming a core developer often involves taking ownership of one of these area
-labels.
-
-## What makes a great Zulip contributor?
-
-Zulip runs a lot of [internship programs](#internship-programs), so we have
-a lot of experience with new contributors. In our experience, these are the
-best predictors of success:
-
-* Posting good questions. This generally means explaining your current
-  understanding, saying what you've done or tried so far, and including
-  tracebacks or other error messages if appropriate.
-* Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
-* Submitting carefully tested code. This generally means checking your work
-  through a combination of automated tests and manually clicking around the
-  UI trying to find bugs in your work. See
-  [things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
-  for additional ideas.
-* Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-* Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-* Being helpful and friendly on chat.zulip.org.
-
-These are also the main criteria we use to select interns for all of our
-internship programs.
-
-## Reporting issues
-
-If you find an easily reproducible bug and/or are experienced in reporting
-bugs, feel free to just open an issue on the relevant project on GitHub.
-
-If you have a feature request or are not yet sure what the underlying bug
-is, the best place to post issues is
-[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
-[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
-[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
-This allows us to interactively figure out what is going on, let you know if
-a similar issue has already been opened, and collect any other information
-we need. Choose a 2-4 word topic that describes the issue, explain the issue
-and how to reproduce it if known, your browser/OS if relevant, and a
-[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-if appropriate.
-
-**Reporting security issues**. Please do not report security issues
-  publicly, including on public streams on chat.zulip.org. You can email
-  zulip-security@googlegroups.com. We create a CVE for every security issue.
-
-## User feedback
-
-Nearly every feature we develop starts with a user request. If you are part
-of a group that is either using or considering using Zulip, we would love to
-hear about your experience with the product. If you're not sure what to
-write, here are some questions we're always very curious to know the answer
-to:
-
-* Evaluation: What is the process by which your organization chose or will
-  choose a group chat product?
-* Pros and cons: What are the pros and cons of Zulip for your organization,
-  and the pros and cons of other products you are evaluating?
-* Features: What are the features that are most important for your
-  organization? In the best case scenario, what would your chat solution do
-  for you?
-* Onboarding: If you remember it, what was your impression during your first
-  few minutes of using Zulip? What did you notice, and how did you feel? Was
-  there anything that stood out to you as confusing, or broken, or great?
-* Organization: What does your organization do? How big is the organization?
-  A link to your organization's website?
-
-## Internship programs
-
-Zulip runs internship programs with
-[Outreachy](https://www.outreachy.org/),
-[Google Summer of Code (GSoC)](https://developers.google.com/open-source/gsoc/)
-[1], and the
-[MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram),
-and has in the past taken summer interns from Harvard, MIT, and
-Stanford.
-
-While each third-party program has its own rules and requirements, the
-Zulip community's approaches all of these programs with these ideas in
-mind:
-* We try to make the application process as valuable for the applicant as
-  possible. Expect high quality code reviews, a supportive community, and
-  publicly viewable patches you can link to from your resume, regardless of
-  whether you are selected.
-* To apply, you'll have to submit at least one pull request to a Zulip
-  repository.  Most students accepted to one of our programs have
-  several merged pull requests (including at least one larger PR) by
-  the time of the application deadline.
-* The main criteria we use is quality of your best contributions, and
-  the bullets listed at
-  [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-  Because we focus on evaluating your best work, it doesn't hurt your
-  application to makes mistakes in your first few PRs as long as your
-  work improves.
-
-Zulip also participates in
-[Google Code-In](https://developers.google.com/open-source/gci/). Our
-selection criteria for Finalists and Grand Prize Winners is the same as our
-selection criteria for interns above.
-
-Most of our interns end up sticking around the project long-term, and many
-quickly become core team members. We hope you apply!
-
-### Google Summer of Code
-
-GSoC is by far the largest of our internship programs (we had 14 GSoC
-students in summer 2017).  While we don't control how many slots
-Google allocates to Zulip, we hope to mentor a similar number of
-students in 2018.
-
-If you're reading this well before the application deadline and want
-to make your application strong, we recommend getting involved in the
-community and fixing issues in Zulip now. Having good contributions
-and building a reputation for doing good work is best way to have a
-strong application.  About half of Zulip's GSoC students for Summer
-2017 had made significant contributions to the project by February
-2017, and about half had not.  Our
-[GSoC project ideas page][gsoc-guide] has lots more details on how
-Zulip does GSoC, as well as project ideas (though the project idea
-list is maintained only during the GSoC application period, so if
-you're looking at some other time of year, the project list is likely
-out-of-date).
-
-We also have in some past years run a Zulip Summer of Code (ZSoC)
-program for students who we didn't have enough slots to accept for
-GSoC but were able to find funding for.  Student expectations are the
-same as with GSoC, and it has no separate application process; your
-GSoC application is your ZSoC application.  If we'd like to select you
-for ZSoC, we'll contact you when the GSoC results are announced.
-
-[gsoc-guide]: https://zulip.readthedocs.io/en/latest/overview/gsoc-ideas.html
-[gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
-
-[1] Formally, [GSoC isn't an internship][gsoc-faq], but it is similar
-enough that we're treating it as such for the purposes of this
-documentation.
-
-## Zulip Outreach
-
-**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
-perception of projects like Zulip. We've collected a few sites below
-where we know Zulip has been discussed. Doing everything in the following
-list typically takes about 15 minutes.
-* Star us on GitHub. There are four main repositories:
-  [server/web](https://github.com/zulip/zulip),
-  [mobile](https://github.com/zulip/zulip-mobile),
-  [desktop](https://github.com/zulip/zulip-electron), and
-  [Python API](https://github.com/zulip/python-zulip-api).
-* [Follow us](https://twitter.com/zulip) on Twitter.
-
-For both of the following, you'll need to make an account on the site if you
-don't already have one.
-
-* [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
-  AlternativeTo. We recommend upvoting a couple of other products you like
-  as well, both to give back to their community, and since single-upvote
-  accounts are generally given less weight. You can also
-  [upvote Zulip](https://alternativeto.net/software/slack/) on their page
-  for Slack.
-* [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
-  it, and upvote the reasons why people like Zulip that you find most
-  compelling. Again, we recommend adding a few other products that you like
-  as well.
-
-We have a doc with more detailed instructions and a few other sites, if you
-have been using Zulip for a while and want to contribute more.
-
-**Blog posts**. Writing a blog post about your experiences with Zulip, or
-about a technical aspect of Zulip can be a great way to spread the word
-about Zulip.
-
-We also occasionally [publish](http://blog.zulip.org/) longer form
-articles related to Zulip. Our posts typically get tens of thousands
-of views, and we always have good ideas for blog posts that we can
-outline but don't have time to write. If you are an experienced writer
-or copyeditor, send us a portfolio; we'd love to talk!

Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN locale-gen en_US.UTF-8
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
+
+WORKDIR /srv/zulip

Dockerfile-dev

@@ -1,17 +0,0 @@
-FROM ubuntu:trusty
-
-EXPOSE 9991
-
-RUN apt-get update && apt-get install -y wget
-
-RUN locale-gen en_US.UTF-8
-
-RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER zulip
-
-RUN ln -nsf /srv/zulip ~/zulip
-
-RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
-
-WORKDIR /srv/zulip

Dockerfile-postgresql

@@ -1,42 +0,0 @@
-# To build run `docker build -f Dockerfile-postgresql .` from the root of the
-# zulip repo.
-
-# Install build tools and build tsearch_extras for the current postgres
-# version. Currently the postgres images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image can not currently be upgraded
-# without users needing a manual pgdump and restore.
-FROM postgres:10
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        postgresql-server-dev-$PG_MAJOR \
-        postgresql-server-dev-all \
-        git \
-        build-essential \
-        fakeroot \
-        devscripts
-RUN git clone https://github.com/zulip/tsearch_extras.git \
-    && cd tsearch_extras \
-    && echo $PG_MAJOR > debian/pgversions \
-    && pg_buildext updatecontrol \
-    && debuild -b -uc -us
-
-# Install tsearch_extras, hunspell, zulip stop words, and run zulip database
-# init.
-FROM postgres:10
-ENV TSEARCH_EXTRAS_VERSION=0.4
-ENV TSEARCH_EXTRAS_DEB=postgresql-${PG_MAJOR}-tsearch-extras_${TSEARCH_EXTRAS_VERSION}_amd64.deb
-COPY --from=0 /${TSEARCH_EXTRAS_DEB} /tmp
-COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop
-COPY scripts/setup/postgres-create-db /docker-entrypoint-initdb.d/postgres-create-db.sh
-COPY scripts/setup/pgroonga-debian.asc /tmp
-RUN apt-key add /tmp/pgroonga-debian.asc \
-    && echo "deb http://packages.groonga.org/debian/ stretch main" > /etc/apt/sources.list.d/zulip.list \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-       hunspell-en-us \
-       postgresql-${PG_MAJOR}-pgroonga \
-    && DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/${TSEARCH_EXTRAS_DEB} \
-    && rm /tmp/${TSEARCH_EXTRAS_DEB} \
-    && ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \
-    && ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \
-    && rm -rf /var/lib/apt/lists/*

LICENSE

@@ -1,24 +1,3 @@
-Copyright 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-The software includes some works released by third parties under other
-free and open source licenses. Those works are redistributed under the
-license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.
-
-
---------------------------------------------------------------------------------
 
                                  Apache License
                            Version 2.0, January 2004
@@ -196,3 +175,28 @@ see the ``docs/THIRDPARTY`` file included with this distribution.
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,83 +1,263 @@
+**[Zulip overview](#zulip-overview)** |
+**[Community](#community)** |
+**[Installing for dev](#installing-the-zulip-development-environment)** |
+**[Installing for production](#running-zulip-in-production)** |
+**[Ways to contribute](#ways-to-contribute)** |
+**[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
+**[License](#license)**
+
 # Zulip overview
 
-Zulip is a powerful, open source group chat application that combines the
-immediacy of real-time chat with the productivity benefits of threaded
-conversations. Zulip is used by open source projects, Fortune 500 companies,
-large standards bodies, and others who need a real-time chat system that
-allows users to easily process hundreds or thousands of messages a day. With
-over 300 contributors merging over 500 commits a month, Zulip is also the
-largest and fastest growing open source group chat project.
+Zulip is a powerful, open source group chat application. Written in
+Python and using the Django framework, Zulip supports both private
+messaging and group chats via conversation streams.
 
-[![CircleCI Build Status](https://circleci.com/gh/zulip/zulip.svg?style=svg)](https://circleci.com/gh/zulip/zulip)
-[![Travis Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip)
-[![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip.svg)](https://codecov.io/gh/zulip/zulip)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
-[![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
+Zulip also supports fast search, drag-and-drop file uploads, image
+previews, group private messages, audible notifications,
+missed-message emails, desktop apps, and much more.
 
-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
+Further information on the Zulip project and its features can be found
+at https://www.zulip.org.
 
-## Getting started
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master) [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/) [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
 
-Click on the appropriate link below. If nothing seems to apply,
-join us on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-and tell us what's up!
+## Community
 
-You might be interested in:
+There are several places online where folks discuss Zulip.
 
-* **Contributing code**. Check out our
-  [guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
-  to get started. Zulip prides itself on maintaining a clean and
-  well-tested codebase, and a stock of hundreds of
-  [beginner-friendly issues][beginner-friendly].
+One of those places is our [public Zulip instance](https://chat.zulip.org/).
+You can go through the simple signup process at that link, and then you
+will soon be talking to core Zulip developers and other users.  To get
+help in real time, you will have the best luck finding core developers
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get a reply
+within minutes to a few hours, depending on time of day.
 
-* **Contributing non-code**.
-  [Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issue),
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
-  into your language,
-  [write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
-  for the Zulip blog, or
-  [give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
-  would love to hear from you, even if you're just trying the product out.
+For Google Summer of Code students and applicants, we have [a mailing
+list](https://groups.google.com/forum/#!forum/zulip-gsoc) for help,
+questions, and announcements.
 
-* **Supporting Zulip**. Advocate for your organization to use Zulip, write a
-  review in the mobile app stores, or
-  [upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
-  product comparison sites.
+We have
+[a public mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
+that is currently pretty low traffic because most discussions happen
+in our public Zulip instance.  We use it to announce Zulip developer
+community gatherings and ask for feedback on major technical or design
+decisions.  It has several hundred subscribers, so you can use it to
+ask questions about features or possible bugs, but please don't use it
+ask for generic help getting started as a contributor (e.g. because
+you want to do Google Summer of Code).  The rest of this page covers
+how to get involved in the Zulip project in detail.
 
-* **Checking Zulip out**. The best way to see Zulip in action is to drop by
-  the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
-  also recommend reading Zulip for
-  [open source](https://zulipchat.com/for/open-source/), Zulip for
-  [companies](https://zulipchat.com/for/companies/), or Zulip for
-  [working groups and part time communities](https://zulipchat.com/for/working-groups-and-communities/).
+Zulip also has a [blog](https://blog.zulip.org/).
 
-* **Running a Zulip server**. Setting up a server takes just a couple
-  of minutes. Zulip runs on Ubuntu 18.04 Bionic, Ubuntu 16.04 Xenial,
-  Ubuntu 14.04 Trusty, and Debian 9 Stretch. The installation process is
-  [documented here](https://zulip.readthedocs.io/en/stable/prod.html).
-  Commercial support is available; see <https://zulipchat.com/plans>
-  for details.
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
+track Zulip-related issues (and store our code, of course).
+Anybody with a GitHub account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull
+Requests as our primary mechanism to receive code contributions.
 
-* **Using Zulip without setting up a server**. <https://zulipchat.com> offers
-  free and commercial hosting.
+The Zulip community has a [Code of Conduct][code-of-conduct].
 
-* **Applying for a Zulip internship**. Zulip runs internship programs with
-  [Outreachy](https://www.outreachy.org/),
-  [Google Summer of Code](https://developers.google.com/open-source/gsoc/),
-  and the
-  [MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram). Zulip
-  also participates in
-  [Google Code-In](https://developers.google.com/open-source/gci/). More
-  information is available
-  [here](https://zulip.readthedocs.io/en/latest/overview/contributing.html#internship-programs).
+## Installing the Zulip Development environment
 
-You may also be interested in reading our [blog](http://blog.zulip.org/) or
-following us on [twitter](https://twitter.com/zulip).
-Zulip is distributed under the
-[Apache 2.0](https://github.com/zulip/zulip/blob/master/LICENSE) license.
+The Zulip development environment is the recommended option for folks
+interested in trying out Zulip.  This is documented in [the developer
+installation guide][dev-install].
 
-[beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22
+## Running Zulip in production
+
+Zulip in production supports Ubuntu 14.04 Trusty and Ubuntu 16.04
+Xenial. Work is ongoing on adding support for additional
+platforms. The installation process is documented at
+https://zulip.org/server.html and in more detail in [the
+documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
+
+## Ways to contribute
+
+Zulip welcomes all forms of contributions!  The page documents the
+Zulip development process.
+
+* **Pull requests**. Before a pull request can be merged, you need to
+sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
+
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+docs][doc-test]. You can also try out [our new desktop
+client][electron], which is in alpha; we'd appreciate testing and
+[feedback](https://github.com/zulip/zulip-electron/issues/new).
+
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat]. You can also improve
+[Zulip.org][z-org].
+
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list](#community) and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list!  Please report any security issues you discover to
+zulip-security@googlegroups.com.
+
+* **App codebases**. This repository is for the Zulip server and web
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].
+
+* **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
+integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
+and [Trello][]), plus [node.js API bindings][node], an [isomorphic
+ JavaScript library][zulip-js], and a [full-text search PostgreSQL
+ extension][tsearch], as separate repos.
+
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation][transifex] if you're interested in
+contributing!
+
+[cla]: https://opensource.dropbox.com/cla/
+[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
+[dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
+[electron]: https://github.com/zulip/zulip-electron
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
+[ios-exp]: https://github.com/zulip/zulip-mobile
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[hubot-adapter]: https://github.com/zulip/hubot-zulip
+[jenkins]: https://github.com/zulip/zulip-jenkins-plugin
+[node]: https://github.com/zulip/zulip-node
+[zulip-js]: https://github.com/zulip/zulip-js
+[phab]: https://github.com/zulip/phabricator-to-zulip
+[puppet]: https://github.com/matthewbarr/puppet-zulip
+[redmine]: https://github.com/zulip/zulip-redmine-plugin
+[trello]: https://github.com/zulip/trello-to-zulip
+[tsearch]: https://github.com/zulip/tsearch_extras
+[transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
+[z-org]: https://github.com/zulip/zulip.github.io
+
+## Google Summer of Code
+
+We participated in
+[GSoC](https://developers.google.com/open-source/gsoc/) last year and
+hope to do so again in 2017.  For guidance, please read
+[our GSoC instructions and ideas page](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
+and feel free to email
+[our GSoC mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc).
+
+## How to get involved with contributing to Zulip
+
+First, subscribe to the Zulip [development discussion mailing
+list](#community).
+
+The Zulip project uses a system of labels in our [issue
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
+
+* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+  The Zulip project loves contributions of new documentation.
+
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
+  The projects that are
+  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
+  These are great projects if you're looking to make an impact.
+
+Another way to find issues in Zulip is to take advantage of our
+"area:<foo>" convention in separating out issues.  We partition all of
+our issues into areas like admin, compose, emoji, hotkeys, i18n,
+onboarding, search, etc.  You can see this here:
+
+[https://github.com/zulip/zulip/labels]
+
+Click on any of the "area:" labels and you will see all the tickets
+related to your area of interest.
+
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.
+
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
+
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
+
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.
+
+Finally, before implementing a larger feature, we highly recommend
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.
+
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just send an email
+to the Zulip Developers list with your thoughts.
+
+## License
+
+Copyright 2011-2016 Dropbox, Inc. and contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The software includes some works released by third parties under other
+free and open source licenses. Those works are redistributed under the
+license terms under which the works were received. For more details,
+see the ``docs/THIRDPARTY`` file included with this distribution.

Vagrantfile

@@ -10,7 +10,7 @@ end
 if Vagrant::VERSION == "1.8.7" then
     path = `which curl`
     if path.include?('/opt/vagrant/embedded/bin/curl') then
-        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 "\
+        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
              "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
              "issue before provisioning. See "\
              "https://github.com/mitchellh/vagrant/issues/7997 "\
@@ -19,53 +19,6 @@ if Vagrant::VERSION == "1.8.7" then
     end
 end
 
-# Workaround: the lxc-config in vagrant-lxc is incompatible with changes in
-# LXC 2.1.0, found in Ubuntu 17.10 artful.  LXC 2.1.1 (in 18.04 LTS bionic)
-# ignores the old config key, so this will only be needed for artful.
-#
-# vagrant-lxc upstream has an attempted fix:
-#   https://github.com/fgrehm/vagrant-lxc/issues/445
-# but it didn't work in our testing.  This is a temporary issue, so we just
-# hack in a fix: we patch the skeleton `lxc-config` file right in the
-# distribution of the vagrant-lxc "box" we use.  If the user doesn't yet
-# have the box (e.g. on first setup), Vagrant would download it but too
-# late for us to patch it like this; so we prompt them to explicitly add it
-# first and then rerun.
-if ['up', 'provision'].include? ARGV[0]
-  if command? "lxc-ls"
-    LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
-    if LXC_VERSION == "2.1.0"
-      lxc_config_file = ENV['HOME'] + "/.vagrant.d/boxes/fgrehm-VAGRANTSLASH-trusty64-lxc/1.2.0/lxc/lxc-config"
-      if File.file?(lxc_config_file)
-        lines = File.readlines(lxc_config_file)
-        deprecated_line = "lxc.pivotdir = lxc_putold\n"
-        if lines[1] == deprecated_line
-          lines[1] = "# #{deprecated_line}"
-          File.open(lxc_config_file, 'w') do |f|
-            f.puts(lines)
-          end
-        end
-      else
-        puts 'You are running LXC 2.1.0, and fgrehm/trusty64-lxc box is incompatible '\
-            "with it by default. First add the box by doing:\n"\
-            "  vagrant box add  https://vagrantcloud.com/fgrehm/trusty64-lxc\n"\
-            'Once this command succeeds, do "vagrant up" again.'
-        exit
-      end
-    end
-  end
-end
-
-# Workaround: Vagrant removed the atlas.hashicorp.com to
-# vagrantcloud.com redirect in February 2018. The value of
-# DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
-# atlas.hashicorp.com, which means that removal broke the fetching and
-# updating of boxes (since the old URL doesn't work).  See
-# https://github.com/hashicorp/vagrant/issues/9442
-if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
-  Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
-end
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # For LXC. VirtualBox hosts use a different box, described below.
@@ -73,17 +26,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # The Zulip development environment runs on 9991 on the guest.
   host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
+  http_proxy = https_proxy = no_proxy = ""
   host_ip_addr = "127.0.0.1"
 
   config.vm.synced_folder ".", "/vagrant", disabled: true
-  if (/darwin/ =~ RUBY_PLATFORM) != nil
-    config.vm.synced_folder ".", "/srv/zulip", type: "nfs",
-        linux__nfs_options: ['rw']
-    config.vm.network "private_network", type: "dhcp"
-  else
-    config.vm.synced_folder ".", "/srv/zulip"
-  end
+  config.vm.synced_folder ".", "/srv/zulip"
 
   vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
   if File.file?(vagrant_config_file)
@@ -101,39 +48,29 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
+  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
+
   if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
+    if http_proxy != ""
       config.proxy.http = http_proxy
     end
-    if !https_proxy.nil?
+    if https_proxy != ""
       config.proxy.https = https_proxy
     end
-    if !no_proxy.nil?
+    if https_proxy != ""
       config.proxy.no_proxy = no_proxy
     end
-  elsif !http_proxy.nil? or !https_proxy.nil?
-    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
-    # We haven't figured out a workaround.
-    puts 'You have specified value for proxy in ~/.zulip-vagrant-config file but did not ' \
-         'install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install ' \
-         'vagrant-proxyconf` in a terminal.  This error will appear twice.'
-    exit
   end
 
-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
   # Specify LXC provider before VirtualBox provider so it's preferred.
   config.vm.provider "lxc" do |lxc|
     if command? "lxc-ls"
       LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
-      if LXC_VERSION >= "1.1.0" and LXC_VERSION < "3.0.0"
+      if LXC_VERSION >= "1.1.0"
         # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
         # see https://github.com/fgrehm/vagrant-lxc/issues/333
         lxc.customize 'aa_allow_incomplete', 1
       end
-      if LXC_VERSION >= "3.0.0"
-        lxc.customize 'apparmor.allow_incomplete', 1
-      end
       if LXC_VERSION >= "2.0.0"
         lxc.backingstore = 'dir'
       end
@@ -157,66 +94,13 @@ $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
-
-# Code should go here, rather than tools/provision, only if it is
-# something that we don't want to happen when running provision in a
-# development environment not using Vagrant.
-
-# Set the MOTD on the system to have Zulip instructions
-sudo rm -f /etc/update-motd.d/*
-sudo bash -c 'cat << EndOfMessage > /etc/motd
-Welcome to the Zulip development environment!  Popular commands:
-* tools/provision - Update the development environment
-* tools/run-dev.py - Run the development server
-* tools/lint - Run the linter (quick and catches many problems)
-* tools/test-* - Run tests (use --help to learn about options)
-
-Read https://zulip.readthedocs.io/en/latest/testing/testing.html to learn
-how to run individual test suites so that you can get a fast debug cycle.
-
-EndOfMessage'
-
 # If the host is running SELinux remount the /sys/fs/selinux directory as read only,
 # needed for apt-get to work.
 if [ -d "/sys/fs/selinux" ]; then
-    sudo mount -o remount,ro /sys/fs/selinux
+  sudo mount -o remount,ro /sys/fs/selinux
 fi
-
-# Set default locale, this prevents errors if the user has another locale set.
-if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
-    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
-fi
-
-# Set an environment variable, so that we won't print the virtualenv
-# shell warning (it'll be wrong, since the shell is dying anyway)
-export SKIP_VENV_SHELL_WARNING=1
-
-# End `set -x`, so that the end of provision doesn't look like an error
-# message after a successful run.
-set +x
-
-# Check if the zulip directory is writable
-if [ ! -w /srv/zulip ]; then
-    echo "The vagrant user is unable to write to the zulip directory."
-    echo "To fix this, run the following commands on the host machine:"
-    # sudo is required since our uid is not 1000
-    echo '    vagrant halt -f'
-    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
-    echo '    sudo chown -R 1000:$(whoami) /PATH/TO/ZULIP/CLONE'
-    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
-    echo "You can resume setting up your vagrant environment by running:"
-    echo "    vagrant up"
-    exit 1
-fi
-# Provision the development environment
 ln -nsf /srv/zulip ~/zulip
 /srv/zulip/tools/provision
-
-# Run any custom provision hooks the user has configured
-if [ -f /srv/zulip/tools/custom_provision ]; then
-    chmod +x /srv/zulip/tools/custom_provision
-    /srv/zulip/tools/custom_provision
-fi
 SCRIPT
 
   config.vm.provision "shell",

analytics/lib/counts.py

@@ -1,86 +1,71 @@
-import time
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
-import logging
-from typing import Any, Callable, Dict, List, \
-    Optional, Tuple, Type, Union
-
-from django.conf import settings
 from django.db import connection, models
-from django.db.models import F
+from django.utils import timezone
+from django.conf import settings
+from datetime import timedelta, datetime
 
-from analytics.models import Anomaly, BaseCount, \
-    FillState, InstallationCount, RealmCount, StreamCount, \
-    UserCount, installation_epoch, last_successful_fill
-from zerver.lib.logging_util import log_to_file
-from zerver.lib.timestamp import ceiling_to_day, \
-    ceiling_to_hour, floor_to_hour, verify_UTC
-from zerver.models import Message, Realm, \
-    Stream, UserActivityInterval, UserProfile, models
+from analytics.models import InstallationCount, RealmCount, \
+    UserCount, StreamCount, BaseCount, FillState, installation_epoch
+from zerver.models import Realm, UserProfile, Message, Stream, models
+from zerver.lib.timestamp import floor_to_day
+
+from typing import Any, Optional, Type, Tuple, Text
+
+import logging
+import time
 
 ## Logging setup ##
+log_format = '%(asctime)s %(levelname)-8s %(message)s'
+logging.basicConfig(format=log_format)
 
-logger = logging.getLogger('zulip.management')
-log_to_file(logger, settings.ANALYTICS_LOG_PATH)
+formatter = logging.Formatter(log_format)
+file_handler = logging.FileHandler(settings.ANALYTICS_LOG_PATH)
+file_handler.setFormatter(formatter)
 
-# You can't subtract timedelta.max from a datetime, so use this instead
-TIMEDELTA_MAX = timedelta(days=365*1000)
+logger = logging.getLogger("zulip.management")
+logger.setLevel(logging.INFO)
+logger.addHandler(file_handler)
 
-## Class definitions ##
+# First post office in Boston
+MIN_TIME = datetime(1639, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
 
-class CountStat:
+class CountStat(object):
     HOUR = 'hour'
     DAY = 'day'
     FREQUENCIES = frozenset([HOUR, DAY])
+    # Allowed intervals are HOUR, DAY, and, GAUGE
+    GAUGE = 'gauge'
 
-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta]=None) -> None:
+    def __init__(self, property, zerver_count_query, filter_args, group_by, frequency, is_gauge):
+        # type: (str, ZerverCountQuery, Dict[str, bool], Optional[Tuple[models.Model, str]], str, bool) -> None
         self.property = property
-        self.data_collector = data_collector
+        self.zerver_count_query = zerver_count_query
         # might have to do something different for bitfields
+        self.filter_args = filter_args
+        self.group_by = group_by
         if frequency not in self.FREQUENCIES:
-            raise AssertionError("Unknown frequency: %s" % (frequency,))
+            raise ValueError("Unknown frequency: %s" % (frequency,))
         self.frequency = frequency
-        if interval is not None:
-            self.interval = interval
-        elif frequency == CountStat.HOUR:
-            self.interval = timedelta(hours=1)
-        else:  # frequency == CountStat.DAY
-            self.interval = timedelta(days=1)
+        self.interval = self.GAUGE if is_gauge else frequency
 
-    def __str__(self) -> str:
-        return "<CountStat: %s>" % (self.property,)
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<CountStat: %s>" % (self.property,)
 
-class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
-        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
+class ZerverCountQuery(object):
+    def __init__(self, zerver_table, analytics_table, query):
+        # type: (Type[models.Model], Type[BaseCount], Text) -> None
+        self.zerver_table = zerver_table
+        self.analytics_table = analytics_table
+        self.query = query
 
-class DependentCountStat(CountStat):
-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta]=None, dependencies: List[str]=[]) -> None:
-        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
-        self.dependencies = dependencies
-
-class DataCollector:
-    def __init__(self, output_table: Type[BaseCount],
-                 pull_function: Optional[Callable[[str, datetime, datetime], int]]) -> None:
-        self.output_table = output_table
-        self.pull_function = pull_function
-
-## CountStat-level operations ##
-
-def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
-    if stat.frequency == CountStat.HOUR:
-        time_increment = timedelta(hours=1)
-    elif stat.frequency == CountStat.DAY:
-        time_increment = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
-
-    verify_UTC(fill_to_time)
-    if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
+def do_update_fill_state(fill_state, end_time, state):
+    # type: (FillState, datetime, int) -> None
+    fill_state.end_time = end_time
+    fill_state.state = state
+    fill_state.save()
 
+def process_count_stat(stat, fill_to_time):
+    # type: (CountStat, datetime) -> None
     fill_state = FillState.objects.filter(property=stat.property).first()
     if fill_state is None:
         currently_filled = installation_epoch()
@@ -90,90 +75,86 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
         logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
     elif fill_state.state == FillState.STARTED:
         logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
-        do_delete_counts_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - time_increment
+        do_delete_count_stat_at_hour(stat, fill_state.end_time)
+        currently_filled = fill_state.end_time - timedelta(hours = 1)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         logger.info("UNDO DONE %s" % (stat.property,))
     elif fill_state.state == FillState.DONE:
         currently_filled = fill_state.end_time
     else:
-        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
+        raise ValueError("Unknown value for FillState.state: %s." % (fill_state.state,))
 
-    if isinstance(stat, DependentCountStat):
-        for dependency in stat.dependencies:
-            dependency_fill_time = last_successful_fill(dependency)
-            if dependency_fill_time is None:
-                logger.warning("DependentCountStat %s run before dependency %s." %
-                               (stat.property, dependency))
-                return
-            fill_to_time = min(fill_to_time, dependency_fill_time)
-
-    currently_filled = currently_filled + time_increment
+    currently_filled = currently_filled + timedelta(hours = 1)
     while currently_filled <= fill_to_time:
-        logger.info("START %s %s" % (stat.property, currently_filled))
+        logger.info("START %s %s %s" % (stat.property, stat.interval, currently_filled))
         start = time.time()
         do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
         do_fill_count_stat_at_hour(stat, currently_filled)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
-        currently_filled = currently_filled + time_increment
-        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
+        currently_filled = currently_filled + timedelta(hours = 1)
+        logger.info("DONE %s %s (%dms)" % (stat.property, stat.interval, (end-start)*1000))
 
-def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
+# We assume end_time is on an hour boundary, and is timezone aware.
+# It is the caller's responsibility to enforce this!
+def do_fill_count_stat_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
+    if stat.frequency == CountStat.DAY and (end_time != floor_to_day(end_time)):
+        return
 
-# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is timezone aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(stat: CountStat, end_time: datetime) -> None:
-    start_time = end_time - stat.interval
-    if not isinstance(stat, LoggingCountStat):
-        timer = time.time()
-        assert(stat.data_collector.pull_function is not None)
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
-        logger.info("%s run pull_function (%dms/%sr)" %
-                    (stat.property, (time.time()-timer)*1000, rows_added))
+    if stat.interval == CountStat.HOUR:
+        start_time = end_time - timedelta(hours = 1)
+    elif stat.interval == CountStat.DAY:
+        start_time = end_time - timedelta(days = 1)
+    else: # stat.interval == CountStat.GAUGE
+        start_time = MIN_TIME
+
+    do_pull_from_zerver(stat, start_time, end_time)
     do_aggregate_to_summary_table(stat, end_time)
 
-def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
-    if isinstance(stat, LoggingCountStat):
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.output_table in [UserCount, StreamCount]:
-            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-    else:
-        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
+def do_delete_count_stat_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
+    UserCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    StreamCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    RealmCount.objects.filter(property = stat.property, end_time = end_time).delete()
+    InstallationCount.objects.filter(property = stat.property, end_time = end_time).delete()
 
-def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
+def do_drop_all_analytics_tables():
+    # type: () -> None
+    UserCount.objects.all().delete()
+    StreamCount.objects.all().delete()
+    RealmCount.objects.all().delete()
+    InstallationCount.objects.all().delete()
+    FillState.objects.all().delete()
+
+def do_aggregate_to_summary_table(stat, end_time):
+    # type: (CountStat, datetime) -> None
     cursor = connection.cursor()
 
     # Aggregate into RealmCount
-    output_table = stat.data_collector.output_table
-    if output_table in (UserCount, StreamCount):
+    analytics_table = stat.zerver_count_query.analytics_table
+    if analytics_table in (UserCount, StreamCount):
         realmcount_query = """
             INSERT INTO analytics_realmcount
                 (realm_id, value, property, subgroup, end_time)
             SELECT
-                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
-                %(output_table)s.subgroup, %%(end_time)s
+                zerver_realm.id, COALESCE(sum(%(analytics_table)s.value), 0), '%(property)s',
+                %(analytics_table)s.subgroup, %%(end_time)s
             FROM zerver_realm
-            JOIN %(output_table)s
+            JOIN %(analytics_table)s
             ON
-                zerver_realm.id = %(output_table)s.realm_id
-            WHERE
-                %(output_table)s.property = '%(property)s' AND
-                %(output_table)s.end_time = %%(end_time)s
-            GROUP BY zerver_realm.id, %(output_table)s.subgroup
-        """ % {'output_table': output_table._meta.db_table,
+            (
+                %(analytics_table)s.realm_id = zerver_realm.id AND
+                %(analytics_table)s.property = '%(property)s' AND
+                %(analytics_table)s.end_time = %%(end_time)s
+            )
+            GROUP BY zerver_realm.id, %(analytics_table)s.subgroup
+        """ % {'analytics_table': analytics_table._meta.db_table,
                'property': stat.property}
         start = time.time()
         cursor.execute(realmcount_query, {'end_time': end_time})
         end = time.time()
-        logger.info("%s RealmCount aggregation (%dms/%sr)" % (
-            stat.property, (end - start) * 1000, cursor.rowcount))
+        logger.info("%s RealmCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
 
     # Aggregate into InstallationCount
     installationcount_query = """
@@ -183,274 +164,85 @@ def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
             sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
         FROM analytics_realmcount
         WHERE
+        (
             property = '%(property)s' AND
             end_time = %%(end_time)s
-        GROUP BY analytics_realmcount.subgroup
+        ) GROUP BY analytics_realmcount.subgroup
     """ % {'property': stat.property}
     start = time.time()
     cursor.execute(installationcount_query, {'end_time': end_time})
     end = time.time()
-    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (
-        stat.property, (end - start) * 1000, cursor.rowcount))
+    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
     cursor.close()
 
-## Utility functions called from outside counts.py ##
-
-# called from zerver/lib/actions.py; should not throw any errors
-def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream], stat: CountStat,
-                              subgroup: Optional[Union[str, int, bool]], event_time: datetime,
-                              increment: int=1) -> None:
-    table = stat.data_collector.output_table
-    if table == RealmCount:
-        id_args = {'realm': zerver_object}
-    elif table == UserCount:
-        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
-    else:  # StreamCount
-        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
-
-    if stat.frequency == CountStat.DAY:
-        end_time = ceiling_to_day(event_time)
-    else:  # CountStat.HOUR:
-        end_time = ceiling_to_hour(event_time)
-
-    row, created = table.objects.get_or_create(
-        property=stat.property, subgroup=subgroup, end_time=end_time,
-        defaults={'value': increment}, **id_args)
-    if not created:
-        row.value = F('value') + increment
-        row.save(update_fields=['value'])
-
-def do_drop_all_analytics_tables() -> None:
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-    Anomaly.objects.all().delete()
-
-def do_drop_single_stat(property: str) -> None:
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-## DataCollector-level operations ##
-
-def do_pull_by_sql_query(property: str, start_time: datetime, end_time: datetime, query: str,
-                         group_by: Optional[Tuple[models.Model, str]]) -> int:
-    if group_by is None:
+# This is the only method that hits the prod databases directly.
+def do_pull_from_zerver(stat, start_time, end_time):
+    # type: (CountStat, datetime, datetime) -> None
+    zerver_table = stat.zerver_count_query.zerver_table._meta.db_table # type: ignore
+    join_args = ' '.join('AND %s.%s = %s' % (zerver_table, key, value)
+                         for key, value in stat.filter_args.items())
+    if stat.group_by is None:
         subgroup = 'NULL'
         group_by_clause  = ''
     else:
-        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
+        subgroup = '%s.%s' % (stat.group_by[0]._meta.db_table, stat.group_by[1])
         group_by_clause = ', ' + subgroup
 
-    # We do string replacement here because cursor.execute will reject a
-    # group_by_clause given as a param.
-    # We pass in the datetimes as params to cursor.execute so that we don't have to
-    # think about how to convert python datetimes to SQL datetimes.
-    query_ = query % {'property': property, 'subgroup': subgroup,
-                      'group_by_clause': group_by_clause}
+    # We do string replacement here because passing join_args as a param
+    # may result in problems when running cursor.execute; we do
+    # the string formatting prior so that cursor.execute runs it as sql
+    query_ = stat.zerver_count_query.query % {'zerver_table': zerver_table,
+                                              'property': stat.property,
+                                              'join_args': join_args,
+                                              'subgroup': subgroup,
+                                              'group_by_clause': group_by_clause}
     cursor = connection.cursor()
+    start = time.time()
     cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
-    rowcount = cursor.rowcount
+    end = time.time()
+    logger.info("%s do_pull_from_zerver (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
     cursor.close()
-    return rowcount
 
-def sql_data_collector(output_table: Type[BaseCount], query: str,
-                       group_by: Optional[Tuple[models.Model, str]]) -> DataCollector:
-    def pull_function(property: str, start_time: datetime, end_time: datetime) -> int:
-        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
-    return DataCollector(output_table, pull_function)
-
-def do_pull_minutes_active(property: str, start_time: datetime, end_time: datetime) -> int:
-    user_activity_intervals = UserActivityInterval.objects.filter(
-        end__gt=start_time, start__lt=end_time
-    ).select_related(
-        'user_profile'
-    ).values_list(
-        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
-
-    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
-    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        start = max(start_time, interval_start)
-        end = min(end_time, interval_end)
-        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
-
-    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
-                      end_time=end_time, value=int(seconds // 60))
-            for ids, seconds in seconds_active.items() if seconds >= 60]
-    UserCount.objects.bulk_create(rows)
-    return len(rows)
-
-count_message_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
-        '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_message
-    ON
-        zerver_userprofile.id = zerver_message.sender_id
-    WHERE
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-# Note: ignores the group_by / group_by_clause.
-count_message_type_by_user_query = """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type = 1 THEN 'private_message'
-             WHEN
-                  zerver_recipient.type = 3 THEN 'huddle_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
-
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.pub_date >= %%(time_start)s AND
-            zerver_message.pub_date < %%(time_end)s
-        JOIN zerver_recipient
-        ON
-            zerver_message.recipient_id = zerver_recipient.id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_recipient.type_id = zerver_stream.id
-        GROUP BY
-            zerver_userprofile.realm_id, zerver_userprofile.id,
-            zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-
-# This query joins to the UserProfile table since all current queries that
-# use this also subgroup on UserProfile.is_bot. If in the future there is a
-# stat that counts messages by stream and doesn't need the UserProfile
-# table, consider writing a new query for efficiency.
-count_message_by_stream_query = """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-        zerver_stream.id = zerver_recipient.type_id
-    JOIN zerver_message
-    ON
-        zerver_recipient.id = zerver_message.recipient_id
-    JOIN zerver_userprofile
-    ON
-        zerver_message.sender_id = zerver_userprofile.id
-    WHERE
-        zerver_stream.date_created < %%(time_end)s AND
-        zerver_recipient.type = 2 AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_stream.id %(group_by_clause)s
-"""
-
-# Hardcodes the query needed by active_users:is_bot:day, since that is
-# currently the only stat that uses this.
 count_user_by_realm_query = """
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
+        zerver_realm.id, count(%(zerver_table)s),'%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_realm
     JOIN zerver_userprofile
     ON
-        zerver_realm.id = zerver_userprofile.realm_id
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
+    (
+        zerver_userprofile.realm_id = zerver_realm.id AND
         zerver_userprofile.date_joined >= %%(time_start)s AND
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_userprofile.is_active = TRUE
+        zerver_userprofile.date_joined < %%(time_end)s
+        %(join_args)s
+    )
+    WHERE
+        zerver_realm.date_created < %%(time_end)s
     GROUP BY zerver_realm.id %(group_by_clause)s
 """
+zerver_count_user_by_realm = ZerverCountQuery(UserProfile, RealmCount, count_user_by_realm_query)
 
-# Currently hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in ['user_created', 'user_deactivated', etc].
-# In particular, it's important to ensure that migrations don't cause that to happen.
-check_realmauditlog_by_user_query = """
+# currently .sender_id is only Message specific thing
+count_message_by_user_query = """
     INSERT INTO analytics_usercount
         (user_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realmauditlog ral1
-    JOIN (
-        SELECT modified_user_id, max(event_time) AS max_event_time
-        FROM zerver_realmauditlog
-        WHERE
-            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
-            event_time < %%(time_end)s
-        GROUP BY modified_user_id
-    ) ral2
-    ON
-        ral1.event_time = max_event_time AND
-        ral1.modified_user_id = ral2.modified_user_id
-    JOIN zerver_userprofile
-    ON
-        ral1.modified_user_id = zerver_userprofile.id
-    WHERE
-        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
-"""
-
-check_useractivityinterval_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
+        zerver_userprofile.id, zerver_userprofile.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_userprofile
-    JOIN zerver_useractivityinterval
+    JOIN zerver_message
     ON
-        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
+    (
+        zerver_message.sender_id = zerver_userprofile.id AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s
+        %(join_args)s
+    )
     WHERE
-        zerver_useractivityinterval.end >= %%(time_start)s AND
-        zerver_useractivityinterval.start < %%(time_end)s
+            zerver_userprofile.date_joined < %%(time_end)s
     GROUP BY zerver_userprofile.id %(group_by_clause)s
 """
-
-count_realm_active_humans_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
-    FROM (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = 'active_users_audit:is_bot:day' AND
-            subgroup = 'false' AND
-            end_time = %%(time_end)s
-    ) usercount1
-    JOIN (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = '15day_actives::day' AND
-            end_time = %%(time_end)s
-    ) usercount2
-    ON
-        usercount1.user_id = usercount2.user_id
-    GROUP BY usercount1.realm_id
-"""
+zerver_count_message_by_user = ZerverCountQuery(Message, UserCount, count_message_by_user_query)
 
 # Currently unused and untested
 count_stream_by_realm_query = """
@@ -461,79 +253,101 @@ count_stream_by_realm_query = """
     FROM zerver_realm
     JOIN zerver_stream
     ON
-        zerver_realm.id = zerver_stream.realm_id AND
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
+    (
+        zerver_stream.realm_id = zerver_realm.id AND
         zerver_stream.date_created >= %%(time_start)s AND
         zerver_stream.date_created < %%(time_end)s
+        %(join_args)s
+    )
+    WHERE
+        zerver_realm.date_created < %%(time_end)s
     GROUP BY zerver_realm.id %(group_by_clause)s
 """
+zerver_count_stream_by_realm = ZerverCountQuery(Stream, RealmCount, count_stream_by_realm_query)
 
-## CountStat declarations ##
+# This query violates the count_X_by_Y_query conventions in several ways. One,
+# the X table is not specified by the query name; MessageType is not a zerver
+# table. Two, it ignores the subgroup column in the CountStat object; instead,
+# it uses 'message_type' from the subquery to fill in the subgroup column.
+count_message_type_by_user_query = """
+    INSERT INTO analytics_usercount
+            (realm_id, user_id, value, property, subgroup, end_time)
+    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
+    FROM
+    (
+        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
+        CASE WHEN
+                  zerver_recipient.type != 2 THEN 'private_message'
+             WHEN
+                  zerver_stream.invite_only = TRUE THEN 'private_stream'
+             ELSE 'public_stream'
+        END
+        message_type
 
-count_stats_ = [
-    # Messages Sent stats
-    # Stats that count the number of messages sent in various ways.
-    # These are also the set of stats that read from the Message table.
+        FROM zerver_userprofile
+        JOIN zerver_message
+        ON
+            zerver_message.sender_id = zerver_userprofile.id AND
+            zerver_message.pub_date >= %%(time_start)s AND
+            zerver_message.pub_date < %%(time_end)s
+            %(join_args)s
+        JOIN zerver_recipient
+        ON
+            zerver_recipient.id = zerver_message.recipient_id
+        LEFT JOIN zerver_stream
+        ON
+            zerver_stream.id = zerver_recipient.type_id
+        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
+    ) AS subquery
+    GROUP BY realm_id, id, message_type
+"""
+zerver_count_message_type_by_user = ZerverCountQuery(Message, UserCount, count_message_type_by_user_query)
 
-    CountStat('messages_sent:is_bot:hour',
-              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.HOUR),
-    CountStat('messages_sent:message_type:day',
-              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
-    CountStat('messages_sent:client:day',
-              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
-              CountStat.DAY),
-    CountStat('messages_in_stream:is_bot:day',
-              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
+# Note that this query also joins to the UserProfile table, since all
+# current queries that use this also subgroup on UserProfile.is_bot. If in
+# the future there is a query that counts messages by stream and doesn't need
+# the UserProfile table, consider writing a new query for efficiency.
+count_message_by_stream_query = """
+    INSERT INTO analytics_streamcount
+        (stream_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
+    FROM zerver_stream
+    JOIN zerver_recipient
+    ON
+    (
+        zerver_recipient.type = 2 AND
+        zerver_stream.id = zerver_recipient.type_id
+    )
+    JOIN zerver_message
+    ON
+    (
+        zerver_message.recipient_id = zerver_recipient.id AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s AND
+        zerver_stream.date_created < %%(time_end)s
+        %(join_args)s
+    )
+    JOIN zerver_userprofile
+    ON zerver_userprofile.id = zerver_message.sender_id
+    GROUP BY zerver_stream.id %(group_by_clause)s
+"""
+zerver_count_message_by_stream = ZerverCountQuery(Message, StreamCount, count_message_by_stream_query)
 
-    # Number of Users stats
-    # Stats that count the number of active users in the UserProfile.is_active sense.
-
-    # 'active_users_audit:is_bot:day' is the canonical record of which users were
-    # active on which days (in the UserProfile.is_active sense).
-    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-    CountStat('active_users_audit:is_bot:day',
-              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-    # sum sequence of 'active_users_log:is_bot:day', for any realm that
-    # started after the latter stat was introduced.
-    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
-    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-    # approximation, e.g. if a user is deactivated between the end of the
-    # day and when this stat is run, they won't be counted. However, is the
-    # simplest of the three to inspect by hand.
-    CountStat('active_users:is_bot:day',
-              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
-              CountStat.DAY, interval=TIMEDELTA_MAX),
-
-    # User Activity stats
-    # Stats that measure user activity in the UserActivityInterval sense.
-
-    CountStat('1day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=1)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('15day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
-
-    # Rate limiting stats
-
-    # Used to limit the number of invitation emails sent by a realm
-    LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
-
-    # Dependent stats
-    # Must come after their dependencies.
-
-    # Canonical account of the number of active humans in a realm on each day.
-    DependentCountStat('realm_active_humans::day',
-                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
-                       CountStat.DAY,
-                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
-]
-
-COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
+COUNT_STATS = {
+    'active_users:is_bot:day': CountStat(
+        'active_users:is_bot:day', zerver_count_user_by_realm, {'is_active': True},
+        (UserProfile, 'is_bot'), CountStat.DAY, True),
+    'messages_sent:is_bot:hour': CountStat(
+        'messages_sent:is_bot:hour', zerver_count_message_by_user, {},
+        (UserProfile, 'is_bot'), CountStat.HOUR, False),
+    'messages_sent:message_type:day': CountStat(
+        'messages_sent:message_type:day', zerver_count_message_type_by_user, {},
+        None, CountStat.DAY, False),
+    'messages_sent:client:day': CountStat(
+        'messages_sent:client:day', zerver_count_message_by_user, {},
+        (Message, 'sending_client_id'), CountStat.DAY, False),
+    'messages_sent_to_stream:is_bot:hour': CountStat(
+        'messages_sent_to_stream:is_bot', zerver_count_message_by_stream, {},
+        (UserProfile, 'is_bot'), CountStat.HOUR, False)
+}

analytics/lib/fixtures.py

@@ -1,14 +1,20 @@
+from __future__ import division, absolute_import
+
+from zerver.models import Realm, UserProfile, Stream, Message
+from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+
+from datetime import datetime
 from math import sqrt
 from random import gauss, random, seed
-from typing import List
 
-from analytics.lib.counts import CountStat
+from six.moves import range, zip
 
-def generate_time_series_data(days: int=100, business_hours_base: float=10,
-                              non_business_hours_base: float=10, growth: float=1,
-                              autocorrelation: float=0, spikiness: float=1,
-                              holiday_rate: float=0, frequency: str=CountStat.DAY,
-                              partial_sum: bool=False, random_seed: int=26) -> List[int]:
+def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
+                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
+                              frequency=CountStat.DAY, is_gauge=False, random_seed=26):
+    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
 
@@ -25,7 +31,7 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
         the variance.
     holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
     frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
+    is_gauge -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
     if frequency == CountStat.HOUR:
@@ -43,10 +49,10 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
                       [24*non_business_hours_base] * 2
         holidays = [random() < holiday_rate for i in range(days)]
     else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
+        raise ValueError("Unknown frequency: %s" % (frequency,))
     if length < 2:
-        raise AssertionError("Must be generating at least 2 data points. "
-                             "Currently generating %s" % (length,))
+        raise ValueError("Must be generating at least 2 data points. "
+                         "Currently generating %s" % (length,))
     growth_base = growth ** (1. / (length-1))
     values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
 
@@ -57,7 +63,7 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
 
     values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
               for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-    if partial_sum:
+    if is_gauge:
         for i in range(1, length):
             values[i] = values[i-1] + values[i]
     return [max(v, 0) for v in values]

analytics/lib/time_utils.py

@@ -1,17 +1,15 @@
+from zerver.lib.timestamp import floor_to_hour, floor_to_day, timestamp_to_datetime
+from analytics.lib.counts import CountStat
+
 from datetime import datetime, timedelta
 from typing import List, Optional
 
-from analytics.lib.counts import CountStat
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
-
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(start: datetime, end: datetime, frequency: str,
-               min_length: Optional[int]) -> List[datetime]:
-    verify_UTC(start)
-    verify_UTC(end)
+def time_range(start, end, frequency, min_length):
+    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
     if frequency == CountStat.HOUR:
         end = floor_to_hour(end)
         step = timedelta(hours=1)
@@ -19,7 +17,7 @@ def time_range(start: datetime, end: datetime, frequency: str,
         end = floor_to_day(end)
         step = timedelta(days=1)
     else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
+        raise ValueError("Unknown frequency: %s" % (frequency,))
 
     times = []
     if min_length is not None:

analytics/management/commands/active_user_stats.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from typing import Any
+
+from zerver.models import UserPresence, UserActivity
+from zerver.lib.utils import statsd, statsd_key
+
+from datetime import datetime, timedelta
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = """Sends active user statistics to statsd.
+
+    Run as a cron job that runs every 10 minutes."""
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        # Get list of all active users in the last 1 week
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+
+        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
+
+        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
+        hour_buckets = [0.16, 2, 12, 24, 48, 168]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+
+        for last_presence in users:
+            if last_presence.status == UserPresence.IDLE:
+                known_active = last_presence.timestamp - timedelta(minutes=30)
+            else:
+                known_active = last_presence.timestamp
+
+            for bucket in hour_buckets:
+                if bucket not in user_info[last_presence.user_profile.realm.string_id]:
+                    user_info[last_presence.user_profile.realm.string_id][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.string_id][bucket].append(last_presence.user_profile.email)
+
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
+
+        # Also do stats for how many users have been reading the app.
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
+        user_info = defaultdict(dict)
+        for activity in users_reading:
+            for bucket in hour_buckets:
+                if bucket not in user_info[activity.user_profile.realm.string_id]:
+                    user_info[activity.user_profile.realm.string_id][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.string_id][bucket].append(activity.user_profile.email)
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers reading for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))

analytics/management/commands/active_user_stats_by_day.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import datetime
+import pytz
+
+from optparse import make_option
+from typing import Any
+from django.core.management.base import BaseCommand, CommandParser
+from zerver.lib.statistics import activity_averages_during_day
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity for a given day."
+
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
+        parser.add_argument('--date', default=None, action='store',
+                            help="Day to query in format 2013-12-05.  Default is yesterday")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options["date"] is None:
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")

analytics/management/commands/analyze_mit.py

@@ -1,19 +1,23 @@
-import datetime
-import logging
-import time
-from typing import Any, Dict
+from __future__ import absolute_import
+from __future__ import print_function
 
+from typing import Any
+
+from optparse import make_option
 from django.core.management.base import BaseCommand, CommandParser
-
+from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Message, Recipient
+import datetime
+import time
+import logging
 
-def compute_stats(log_level: int) -> None:
+def compute_stats(log_level):
+    # type: (int) -> None
     logger = logging.getLogger()
     logger.setLevel(log_level)
 
     one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
+    mit_query = Message.objects.filter(sender__realm__string_id="mit",
                                        recipient__type=Recipient.STREAM,
                                        pub_date__gt=one_week_ago)
     for bot_sender_start in ["imap.", "rcmd.", "sys."]:
@@ -26,15 +30,15 @@ def compute_stats(log_level: int) -> None:
                            "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                            "root@mit.edu", "nagios@mit.edu",
                            "www-data|local-realm@mit.edu"])
-    user_counts = {}  # type: Dict[str, Dict[str, int]]
+    user_counts = {} # type: Dict[str, Dict[str, int]]
     for m in mit_query.select_related("sending_client", "sender"):
         email = m.sender.email
         user_counts.setdefault(email, {})
         user_counts[email].setdefault(m.sending_client.name, 0)
         user_counts[email][m.sending_client.name] += 1
 
-    total_counts = {}  # type: Dict[str, int]
-    total_user_counts = {}  # type: Dict[str, int]
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
     for email, counts in user_counts.items():
         total_user_counts.setdefault(email, 0)
         for client_name, count in counts.items():
@@ -43,7 +47,7 @@ def compute_stats(log_level: int) -> None:
             total_user_counts[email] += count
 
     logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}  # type: Dict[int, float]
+    top_percents = {} # type: Dict[int, float]
     for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
         top_percents[size] = 0.0
     for i, email in enumerate(sorted(total_user_counts.keys(),
@@ -71,10 +75,12 @@ def compute_stats(log_level: int) -> None:
 class Command(BaseCommand):
     help = "Compute statistics on MIT Zephyr usage."
 
-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
         parser.add_argument('--verbose', default=False, action='store_true')
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         level = logging.INFO
         if options["verbose"]:
             level = logging.DEBUG

analytics/management/commands/analyze_user_activity.py

@@ -1,13 +1,19 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 from typing import Any, Dict
 
+from zerver.lib.statistics import seconds_usage_between
+
+from optparse import make_option
 from django.core.management.base import BaseCommand, CommandParser
+from zerver.models import UserProfile
+import datetime
 from django.utils.timezone import utc
 
-from zerver.lib.statistics import seconds_usage_between
-from zerver.models import UserProfile
-
-def analyze_activity(options: Dict[str, Any]) -> None:
+def analyze_activity(options):
+    # type: (Dict[str, Any]) -> None
     day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
     day_end = day_start + datetime.timedelta(days=options["duration"])
 
@@ -46,11 +52,13 @@ Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""
 
-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
         parser.add_argument('--realm', action='store')
         parser.add_argument('--date', action='store', default="2013-09-06")
         parser.add_argument('--duration', action='store', default=1, type=int,
                             help="How many days to show usage information for")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         analyze_activity(options)

analytics/management/commands/check_analytics_state.py

@@ -1,84 +0,0 @@
-from argparse import ArgumentParser
-from datetime import timedelta
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.models import InstallationCount, installation_epoch, \
-    last_successful_fill
-from analytics.lib.counts import COUNT_STATS, CountStat
-from zerver.lib.timestamp import floor_to_hour, floor_to_day, verify_UTC, \
-    TimezoneNotUTCException
-from zerver.models import Realm
-
-import os
-import sys
-import time
-from typing import Any, Dict
-
-states = {
-    0: "OK",
-    1: "WARNING",
-    2: "CRITICAL",
-    3: "UNKNOWN"
-}
-
-class Command(BaseCommand):
-    help = """Checks FillState table.
-
-    Run as a cron job that runs every hour."""
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        fill_state = self.get_fill_state()
-        status = fill_state['status']
-        message = fill_state['message']
-
-        state_file_path = "/var/lib/nagios_state/check-analytics-state"
-        state_file_tmp = state_file_path + "-tmp"
-
-        with open(state_file_tmp, "w") as f:
-            f.write("%s|%s|%s|%s\n" % (
-                int(time.time()), status, states[status], message))
-        os.rename(state_file_tmp, state_file_path)
-
-    def get_fill_state(self) -> Dict[str, Any]:
-        if not Realm.objects.exists():
-            return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
-
-        warning_unfilled_properties = []
-        critical_unfilled_properties = []
-        for property, stat in COUNT_STATS.items():
-            last_fill = last_successful_fill(property)
-            if last_fill is None:
-                last_fill = installation_epoch()
-            try:
-                verify_UTC(last_fill)
-            except TimezoneNotUTCException:
-                return {'status': 2, 'message': 'FillState not in UTC for %s' % (property,)}
-
-            if stat.frequency == CountStat.DAY:
-                floor_function = floor_to_day
-                warning_threshold = timedelta(hours=26)
-                critical_threshold = timedelta(hours=50)
-            else:  # CountStat.HOUR
-                floor_function = floor_to_hour
-                warning_threshold = timedelta(minutes=90)
-                critical_threshold = timedelta(minutes=150)
-
-            if floor_function(last_fill) != last_fill:
-                return {'status': 2, 'message': 'FillState not on %s boundary for %s' %
-                        (stat.frequency, property)}
-
-            time_to_last_fill = timezone_now() - last_fill
-            if time_to_last_fill > critical_threshold:
-                critical_unfilled_properties.append(property)
-            elif time_to_last_fill > warning_threshold:
-                warning_unfilled_properties.append(property)
-
-        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return {'status': 0, 'message': 'FillState looks fine.'}
-        if len(critical_unfilled_properties) == 0:
-            return {'status': 1, 'message': 'Missed filling %s once.' %
-                    (', '.join(warning_unfilled_properties),)}
-        return {'status': 2, 'message': 'Missed filling %s once. Missed filling %s at least twice.' %
-                (', '.join(warning_unfilled_properties), ', '.join(critical_unfilled_properties))}

analytics/management/commands/clear_analytics_tables.py

@@ -1,20 +1,27 @@
-import sys
-from argparse import ArgumentParser
-from typing import Any
+from __future__ import absolute_import
+from __future__ import print_function
 
+import sys
+
+from argparse import ArgumentParser
+from django.db import connection
 from django.core.management.base import BaseCommand
 
 from analytics.lib.counts import do_drop_all_analytics_tables
 
+from typing import Any
+
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--force',
                             action='store_true',
                             help="Clear analytics tables.")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['force']:
             do_drop_all_analytics_tables()
         else:

analytics/management/commands/clear_single_stat.py

@@ -1,29 +0,0 @@
-import sys
-from argparse import ArgumentParser
-from typing import Any
-
-from django.core.management.base import BaseCommand
-
-from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument('--force',
-                            action='store_true',
-                            help="Actually do it.")
-        parser.add_argument('--property',
-                            type=str,
-                            help="The property of the stat to be cleared.")
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        property = options['property']
-        if property not in COUNT_STATS:
-            print("Invalid property: %s" % (property,))
-            sys.exit(1)
-        if not options['force']:
-            print("No action taken. Use --force.")
-            sys.exit(1)
-
-        do_drop_single_stat(property)

analytics/management/commands/client_activity.py

@@ -1,32 +1,33 @@
-import datetime
-from argparse import ArgumentParser
+from __future__ import absolute_import
+from __future__ import print_function
+
 from typing import Any
 
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
-from django.utils.timezone import now as timezone_now
 
-from zerver.lib.management import ZulipBaseCommand
-from zerver.models import UserActivity
+from zerver.models import UserActivity, UserProfile, Realm, \
+    get_realm, get_user_profile_by_email
 
-class Command(ZulipBaseCommand):
+import datetime
+
+class Command(BaseCommand):
     help = """Report rough client activity globally, for a realm, or for a user
 
 Usage examples:
 
-./manage.py client_activity --target server
-./manage.py client_activity --target realm --realm zulip
-./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
+./manage.py client_activity
+./manage.py client_activity zulip
+./manage.py client_activity hamlet@zulip.com"""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument('--target', dest='target', required=True, type=str,
-                            help="'server' will calculate client activity of the entire server. "
-                                 "'realm' will calculate client activity of realm. "
-                                 "'user' will calculate client activity of the user.")
-        parser.add_argument('--user', dest='user', type=str,
-                            help="The email address of the user you want to calculate activity.")
-        self.add_realm_args(parser)
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
+                            help="realm or user to estimate client activity for")
 
-    def compute_activity(self, user_activity_objects: QuerySet) -> None:
+    def compute_activity(self, user_activity_objects):
+        # type: (QuerySet) -> None
         # Report data from the past week.
         #
         # This is a rough report of client activity because we inconsistently
@@ -37,7 +38,7 @@ Usage examples:
         #
         # Importantly, this does NOT tell you anything about the relative
         # volumes of requests from clients.
-        threshold = timezone_now() - datetime.timedelta(days=7)
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
         client_counts = user_activity_objects.filter(
             last_visit__gt=threshold).values("client__name").annotate(
             count=Count('client__name'))
@@ -56,18 +57,24 @@ Usage examples:
             print("%25s %15d" % (count[1], count[0]))
         print("Total:", total)
 
-    def handle(self, *args: Any, **options: str) -> None:
-        realm = self.get_realm(options)
-        if options["user"] is None:
-            if options["target"] == "server" and realm is None:
-                # Report global activity.
-                self.compute_activity(UserActivity.objects.all())
-            elif options["target"] == "realm" and realm is not None:
-                self.compute_activity(UserActivity.objects.filter(user_profile__realm=realm))
-            else:
-                self.print_help("./manage.py", "client_activity")
-        elif options["target"] == "user":
-            user_profile = self.get_user(options["user"], realm)
-            self.compute_activity(UserActivity.objects.filter(user_profile=user_profile))
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['arg'] is None:
+            # Report global activity.
+            self.compute_activity(UserActivity.objects.all())
         else:
-            self.print_help("./manage.py", "client_activity")
+            arg = options['arg']
+            try:
+                # Report activity for a user.
+                user_profile = get_user_profile_by_email(arg)
+                self.compute_activity(UserActivity.objects.filter(
+                    user_profile=user_profile))
+            except UserProfile.DoesNotExist:
+                try:
+                    # Report activity for a realm.
+                    realm = get_realm(arg)
+                    self.compute_activity(UserActivity.objects.filter(
+                        user_profile__realm=realm))
+                except Realm.DoesNotExist:
+                    print("Unknown user or realm %s" % (arg,))
+                    exit(1)

analytics/management/commands/populate_analytics_db.py

@@ -1,20 +1,22 @@
+from __future__ import absolute_import, print_function
 
-from datetime import datetime, timedelta
-from typing import Any, Dict, List, Mapping, Optional, Type, Union
+from argparse import ArgumentParser
 
 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
+from django.utils import timezone
 
-from analytics.lib.counts import COUNT_STATS, \
-    CountStat, do_drop_all_analytics_tables
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount
+from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, FillState, RealmCount, UserCount, \
-    StreamCount, InstallationCount
-from zerver.lib.actions import do_change_is_admin
 from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, UserProfile, Stream, Message, Client, \
-    RealmAuditLog, Recipient
+from zerver.models import Realm, UserProfile, Stream, Message, Client
+
+from datetime import datetime, timedelta
+
+from six.moves import zip
+from typing import Any, List, Optional, Text, Type, Union
 
 class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
@@ -22,194 +24,100 @@ class Command(BaseCommand):
     DAYS_OF_DATA = 100
     random_seed = 26
 
-    def create_user(self, email: str,
-                    full_name: str,
-                    is_staff: bool,
-                    date_joined: datetime,
-                    realm: Realm) -> UserProfile:
-        user = UserProfile.objects.create(
+    def create_user(self, email, full_name, is_staff, date_joined, realm):
+        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
+        return UserProfile.objects.create(
             email=email, full_name=full_name, is_staff=is_staff,
             realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
             api_key='42', date_joined=date_joined)
-        RealmAuditLog.objects.create(
-            realm=realm, modified_user=user, event_type=RealmAuditLog.USER_CREATED,
-            event_time=user.date_joined)
-        return user
 
-    def generate_fixture_data(self, stat: CountStat, business_hours_base: float,
-                              non_business_hours_base: float, growth: float,
-                              autocorrelation: float, spikiness: float,
-                              holiday_rate: float=0, partial_sum: bool=False) -> List[int]:
+    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
+                              growth, autocorrelation, spikiness, holiday_rate=0):
+        # type: (CountStat, float, float, float, float, float, float) -> List[int]
         self.random_seed += 1
         return generate_time_series_data(
             days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
             non_business_hours_base=non_business_hours_base, growth=growth,
             autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
+            frequency=stat.frequency, is_gauge=(stat.interval == CountStat.GAUGE),
+            random_seed=self.random_seed)
 
-    def handle(self, *args: Any, **options: Any) -> None:
-        # TODO: This should arguably only delete the objects
-        # associated with the "analytics" realm.
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         do_drop_all_analytics_tables()
-
-        # This also deletes any objects with this realm as a foreign key
+        # I believe this also deletes any objects with this realm as a foreign key
         Realm.objects.filter(string_id='analytics').delete()
+        Client.objects.filter(name__endswith='_').delete()
 
-        # Because we just deleted a bunch of objects in the database
-        # directly (rather than deleting individual objects in Django,
-        # in which case our post_save hooks would have flushed the
-        # individual objects from memcached for us), we need to flush
-        # memcached in order to ensure deleted objects aren't still
-        # present in the memcached cache.
-        from zerver.apps import flush_cache
-        flush_cache(None)
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
+        installation_time = timezone.now() - timedelta(days=self.DAYS_OF_DATA)
+        last_end_time = floor_to_day(timezone.now())
         realm = Realm.objects.create(
-            string_id='analytics', name='Analytics', date_created=installation_time)
+            string_id='analytics', name='Analytics', domain='analytics.ds',
+            date_created=installation_time)
         shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
-        do_change_is_admin(shylock, True)
-        stream = Stream.objects.create(
-            name='all', realm=realm, date_created=installation_time)
-        Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
 
-        def insert_fixture_data(stat: CountStat,
-                                fixture_data: Mapping[Optional[str], List[int]],
-                                table: Type[BaseCount]) -> None:
+        def insert_fixture_data(stat, fixture_data, table):
+            # type: (CountStat, Dict[Optional[str], List[int]], Type[BaseCount]) -> None
             end_times = time_range(last_end_time, last_end_time, stat.frequency,
                                    len(list(fixture_data.values())[0]))
-            if table == InstallationCount:
-                id_args = {}  # type: Dict[str, Any]
             if table == RealmCount:
                 id_args = {'realm': realm}
             if table == UserCount:
                 id_args = {'realm': realm, 'user': shylock}
-            if table == StreamCount:
-                id_args = {'stream': stream, 'realm': realm}
-
             for subgroup, values in fixture_data.items():
                 table.objects.bulk_create([
                     table(property=stat.property, subgroup=subgroup, end_time=end_time,
                           value=value, **id_args)
                     for end_time, value in zip(end_times, values) if value != 0])
 
-        stat = COUNT_STATS['1day_actives::day']
+        stat = COUNT_STATS['active_users:is_bot:day']
         realm_data = {
-            None: self.generate_fixture_data(stat, .08, .02, 3, .3, 6, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
+            'false': self.generate_fixture_data(stat, .1, .03, 3, .5, 3),
+            'true': self.generate_fixture_data(stat, .01, 0, 1, 0, 1)
+        } # type: Dict[Optional[str], List[int]]
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, .8, .2, 4, .3, 6, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['realm_active_humans::day']
-        realm_data = {
-            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 1, .3, 4, .5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        realm_data = {
-            'false': self.generate_fixture_data(stat, .1, .03, 3.5, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'false': self.generate_fixture_data(stat, 1, .3, 6, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
 
         stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data = {'false': self.generate_fixture_data(
-            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
+        user_data = {'false': self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
                       'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {'false': self.generate_fixture_data(stat, 350, 150, 6, .6, 4),
-                             'true': self.generate_fixture_data(stat, 150, 150, 3, .4, 2)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
 
         stat = COUNT_STATS['messages_sent:message_type:day']
         user_data = {
             'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
-            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
+            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8)}
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
             'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
             'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
+            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'public_stream': self.generate_fixture_data(stat, 300, 80, 5, .6, 4),
-            'private_stream': self.generate_fixture_data(stat, 70, 70, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 130, 50, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, .6, 4)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
 
-        website, created = Client.objects.get_or_create(name='website')
-        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
-        android, created = Client.objects.get_or_create(name='ZulipAndroid')
-        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
-        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
-        API, created = Client.objects.get_or_create(name='API: Python')
-        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
-        unused, created = Client.objects.get_or_create(name='unused')
-        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
+        website_ = Client.objects.create(name='website_')
+        API_ = Client.objects.create(name='API_')
+        android_ = Client.objects.create(name='android_')
+        iOS_ = Client.objects.create(name='iOS_')
+        react_native_ = Client.objects.create(name='react_native_')
+        electron_ = Client.objects.create(name='electron_')
+        barnowl_ = Client.objects.create(name='barnowl_')
+        plan9_ = Client.objects.create(name='plan9_')
 
         stat = COUNT_STATS['messages_sent:client:day']
         user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
+            website_.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
+            barnowl_.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
+            website_.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
+            API_.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
+            android_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            iOS_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
+            react_native_.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
+            electron_.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
+            barnowl_.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
+            plan9_.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0, 0)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            website.id: self.generate_fixture_data(stat, 300, 200, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 50, 50, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
 
-        stat = COUNT_STATS['messages_in_stream:is_bot:day']
-        realm_data = {'false': self.generate_fixture_data(stat, 30, 5, 6, .6, 4),
-                      'true': self.generate_fixture_data(stat, 20, 2, 3, .2, 3)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data = {'false': self.generate_fixture_data(stat, 10, 7, 5, .6, 4),
-                       'true': self.generate_fixture_data(stat, 5, 3, 2, .4, 2)}  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, stream_data, StreamCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
+        # TODO: messages_sent_to_stream:is_bot

analytics/management/commands/realm_stats.py

@@ -1,14 +1,17 @@
-import datetime
-from argparse import ArgumentParser
-from typing import Any, List
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
 
+from typing import Any
+
+from argparse import ArgumentParser
+import datetime
 import pytz
+
 from django.core.management.base import BaseCommand
 from django.db.models import Count
-from django.utils.timezone import now as timezone_now
-
-from zerver.models import Message, Realm, Recipient, Stream, \
-    Subscription, UserActivity, UserMessage, UserProfile, get_realm
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
+    Subscription, UserMessage, get_realm
 
 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
@@ -18,13 +21,15 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
 class Command(BaseCommand):
     help = "Generate statistics on realm activity."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def active_users(self, realm: Realm) -> List[UserProfile]:
+    def active_users(self, realm):
+        # type: (Realm) -> List[UserProfile]
         # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
         return [activity.user_profile for activity in (
             UserActivity.objects.filter(user_profile__realm=realm,
                                         user_profile__is_active=True,
@@ -32,44 +37,53 @@ class Command(BaseCommand):
                                         query="/json/users/me/pointer",
                                         client__name="website"))]
 
-    def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def messages_sent_by(self, user, days_ago):
+        # type: (UserProfile, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
 
-    def total_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def total_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
-    def human_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def human_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
-    def api_messages(self, realm: Realm, days_ago: int) -> int:
+    def api_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
 
-    def stream_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def stream_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                      recipient__type=Recipient.STREAM).count()
 
-    def private_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
 
-    def group_private_messages(self, realm: Realm, days_ago: int) -> int:
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+    def group_private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
         return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
 
-    def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
+    def report_percentage(self, numerator, denominator, text):
+        # type: (float, float, str) -> None
         if not denominator:
             fraction = 0.0
         else:
             fraction = numerator / float(denominator)
         print("%.2f%% of" % (fraction * 100,), text)
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]
@@ -145,7 +159,7 @@ class Command(BaseCommand):
                 len(markup_messages), sum([elt["count"] for elt in markup_messages])))
 
             # Notifications for stream messages
-            notifications = active_user_subs.filter(desktop_notifications=True).values(
+            notifications = active_user_subs.filter(notifications=True).values(
                 "user_profile").annotate(count=Count("user_profile"))
             print("%d users receive desktop notifications for %d streams" % (
                 len(notifications), sum([elt["count"] for elt in notifications])))

analytics/management/commands/stream_stats.py

@@ -1,20 +1,23 @@
-from argparse import ArgumentParser
+from __future__ import absolute_import
+from __future__ import print_function
+
 from typing import Any
 
+from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
 from django.db.models import Q
-
-from zerver.models import Message, Realm, \
-    Recipient, Stream, Subscription, get_realm
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
 
 class Command(BaseCommand):
     help = "Generate statistics on the streams for a realm."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def handle(self, *args: Any, **options: str) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]
@@ -36,9 +39,8 @@ class Command(BaseCommand):
                     continue
                 print("%25s" % (stream.name,), end=' ')
                 recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
-                print("%10d" % (len(Subscription.objects.filter(recipient=recipient,
-                                                                active=True)),), end=' ')
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
                 num_messages = len(Message.objects.filter(recipient=recipient))
                 print("%12d" % (num_messages,))
-            print("%d private streams" % (invite_only_count,))
+            print("%d invite-only streams" % (invite_only_count,))
             print("")

analytics/management/commands/update_analytics_counts.py

@@ -1,43 +1,49 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 import os
-import time
-from argparse import ArgumentParser
-from typing import Any, Dict
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from django.utils.dateparse import parse_datetime
-from django.utils.timezone import now as timezone_now
-from django.utils.timezone import utc as timezone_utc
-
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+import sys
 from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import Realm
+
+from argparse import ArgumentParser
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+from django.conf import settings
+
+from analytics.models import RealmCount, UserCount
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from zerver.lib.timestamp import datetime_to_string, is_timezone_aware
+from zerver.models import UserProfile, Message
+
+from typing import Any
 
 class Command(BaseCommand):
     help = """Fills Analytics tables.
 
     Run as a cron job that runs every hour."""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--time', '-t',
                             type=str,
-                            help='Update stat tables from current state to'
-                                 '--time. Defaults to the current time.',
-                            default=timezone_now().isoformat())
+                            help='Update stat tables from current state to --time. Defaults to the current time.',
+                            default=datetime_to_string(timezone.now()))
         parser.add_argument('--utc',
-                            action='store_true',
+                            type=bool,
                             help="Interpret --time in UTC.",
                             default=False)
         parser.add_argument('--stat', '-s',
                             type=str,
                             help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument('--verbose',
-                            action='store_true',
-                            help="Print timing information to stdout.",
-                            default=False)
+        parser.add_argument('--quiet', '-q',
+                            type=str,
+                            help="Suppress output to stdout.")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         try:
             os.mkdir(settings.ANALYTICS_LOCK_DIR)
         except OSError:
@@ -49,38 +55,21 @@ class Command(BaseCommand):
         finally:
             os.rmdir(settings.ANALYTICS_LOCK_DIR)
 
-    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
-        # installation_epoch relies on there being at least one realm; we
-        # shouldn't run the analytics code if that condition isn't satisfied
-        if not Realm.objects.exists():
-            logger.info("No realms, stopping update_analytics_counts")
-            return
-
+    def run_update_analytics_counts(self, options):
+        # type: (Dict[str, Any]) -> None
         fill_to_time = parse_datetime(options['time'])
         if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
-        if fill_to_time.tzinfo is None:
+            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
+
+        if not (is_timezone_aware(fill_to_time)):
             raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
 
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
+        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
 
         if options['stat'] is not None:
-            stats = [COUNT_STATS[options['stat']]]
+            process_count_stat(COUNT_STATS[options['stat']], fill_to_time)
         else:
-            stats = list(COUNT_STATS.values())
+            for stat in COUNT_STATS.values():
+                process_count_stat(stat, fill_to_time)
 
-        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
-        if options['verbose']:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options['verbose']:
-                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
-                last = time.time()
-
-        if options['verbose']:
-            print("Finished updating analytics counts through %s in %.3fs" %
-                  (fill_to_time, time.time() - start))
         logger.info("Finished updating analytics counts through %s" % (fill_to_time,))

analytics/management/commands/user_stats.py

@@ -1,25 +1,31 @@
-import datetime
+from __future__ import absolute_import
+from __future__ import print_function
+
 from argparse import ArgumentParser
+import datetime
+import pytz
 from typing import Any
 
 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from zerver.models import Message, Realm, Stream, UserProfile, get_realm
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range
 
 class Command(BaseCommand):
     help = "Generate statistics on user activity."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def messages_sent_by(self, user: UserProfile, week: int) -> int:
-        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
-        end = timezone_now() - datetime.timedelta(days=week*7)
+    def messages_sent_by(self, user, week):
+        # type: (UserProfile, int) -> int
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
         return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]

analytics/migrations/0001_initial.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
+from __future__ import unicode_literals
 
+from django.db import models, migrations
+from django.conf import settings
 import zerver.lib.str_utils
 
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -19,21 +20,21 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('info', models.CharField(max_length=1000)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='HuddleCount',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('huddle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('huddle', models.ForeignKey(to='zerver.Recipient')),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                 ('property', models.CharField(max_length=40)),
                 ('end_time', models.DateTimeField()),
                 ('interval', models.CharField(max_length=20)),
                 ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='InstallationCount',
@@ -43,51 +44,51 @@ class Migration(migrations.Migration):
                 ('end_time', models.DateTimeField()),
                 ('interval', models.CharField(max_length=20)),
                 ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='RealmCount',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
                 ('property', models.CharField(max_length=40)),
                 ('end_time', models.DateTimeField()),
                 ('interval', models.CharField(max_length=20)),
                 ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
 
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='StreamCount',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream')),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
+                ('stream', models.ForeignKey(to='zerver.Stream')),
                 ('property', models.CharField(max_length=40)),
                 ('end_time', models.DateTimeField()),
                 ('interval', models.CharField(max_length=20)),
                 ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='UserCount',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('realm', models.ForeignKey(to='zerver.Realm')),
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
                 ('property', models.CharField(max_length=40)),
                 ('end_time', models.DateTimeField()),
                 ('interval', models.CharField(max_length=20)),
                 ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.AlterUniqueTogether(
             name='usercount',

analytics/migrations/0002_remove_huddlecount.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0003_fillstate.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
-from django.db import migrations, models
+from __future__ import unicode_literals
 
+from django.db import migrations, models
 import zerver.lib.str_utils
 
+
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -19,6 +21,6 @@ class Migration(migrations.Migration):
                 ('state', models.PositiveSmallIntegerField()),
                 ('last_modified', models.DateTimeField(auto_now=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0005_alter_field_size.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0007_remove_interval.py

@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-16 20:50
+from __future__ import unicode_literals
+
 from django.conf import settings
 from django.db import migrations
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0008_add_count_indexes.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-02-01 22:28
+from __future__ import unicode_literals
+
 from django.db import migrations
 
+
 class Migration(migrations.Migration):
 
     dependencies = [

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent_to_stream:is_bot'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0008_add_count_indexes'),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent:message_type:day'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0010_clear_messages_sent_values'),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]

analytics/migrations/0012_add_on_delete.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.6 on 2018-01-29 08:14
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0011_clear_analytics_tables'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='installationcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-    ]

analytics/models.py

@@ -1,100 +1,151 @@
-import datetime
-from typing import Any, Dict, Optional, Tuple, Union
-
 from django.db import models
+from django.utils import timezone
 
-from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, Recipient, Stream, UserProfile
+from zerver.models import Realm, UserProfile, Stream, Recipient
+from zerver.lib.str_utils import ModelReprMixin
+from zerver.lib.timestamp import datetime_to_UTC, floor_to_day
 
-class FillState(models.Model):
-    property = models.CharField(max_length=40, unique=True)  # type: str
-    end_time = models.DateTimeField()  # type: datetime.datetime
+import datetime
+
+from typing import Optional, Tuple, Union, Dict, Any, Text
+
+class FillState(ModelReprMixin, models.Model):
+    property = models.CharField(max_length=40, unique=True) # type: Text
+    end_time = models.DateTimeField() # type: datetime.datetime
 
     # Valid states are {DONE, STARTED}
     DONE = 1
     STARTED = 2
-    state = models.PositiveSmallIntegerField()  # type: int
+    state = models.PositiveSmallIntegerField() # type: int
 
-    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
+    last_modified = models.DateTimeField(auto_now=True) # type: datetime.datetime
 
-    def __str__(self) -> str:
-        return "<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime.datetime:
+def installation_epoch():
+    # type: () -> datetime.datetime
     earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    return floor_to_day(earliest_realm_creation)
-
-def last_successful_fill(property: str) -> Optional[datetime.datetime]:
-    fillstate = FillState.objects.filter(property=property).first()
-    if fillstate is None:
-        return None
-    if fillstate.state == FillState.DONE:
-        return fillstate.end_time
-    return fillstate.end_time - datetime.timedelta(hours=1)
+    return floor_to_day(datetime_to_UTC(earliest_realm_creation))
 
 # would only ever make entries here by hand
-class Anomaly(models.Model):
-    info = models.CharField(max_length=1000)  # type: str
+class Anomaly(ModelReprMixin, models.Model):
+    info = models.CharField(max_length=1000) # type: Text
 
-    def __str__(self) -> str:
-        return "<Anomaly: %s... %s>" % (self.info, self.id)
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<Anomaly: %s... %s>" % (self.info, self.id)
 
-class BaseCount(models.Model):
+class BaseCount(ModelReprMixin, models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)  # type: str
-    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[str]
-    end_time = models.DateTimeField()  # type: datetime.datetime
-    value = models.BigIntegerField()  # type: int
-    anomaly = models.ForeignKey(Anomaly, on_delete=models.SET_NULL, null=True)  # type: Optional[Anomaly]
+    property = models.CharField(max_length=32) # type: Text
+    subgroup = models.CharField(max_length=16, null=True) # type: Text
+    end_time = models.DateTimeField() # type: datetime.datetime
+    value = models.BigIntegerField() # type: int
+    anomaly = models.ForeignKey(Anomaly, null=True) # type: Optional[Anomaly]
 
-    class Meta:
+    class Meta(object):
         abstract = True
 
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        raise NotImplementedError
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        raise NotImplementedError
+
 class InstallationCount(BaseCount):
 
-    class Meta:
+    class Meta(object):
         unique_together = ("property", "subgroup", "end_time")
 
-    def __str__(self) -> str:
-        return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ()
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return None
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
 
 class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
         unique_together = ("realm", "property", "subgroup", "end_time")
         index_together = ["property", "end_time"]
 
-    def __str__(self) -> str:
-        return "<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('realm_id',)
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Realm
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
 
 class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    user = models.ForeignKey(UserProfile)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
         unique_together = ("user", "property", "subgroup", "end_time")
         # This index dramatically improves the performance of
         # aggregating from users to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> str:
-        return "<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('user_id', 'realm_id')
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return UserProfile
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
 
 class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    stream = models.ForeignKey(Stream)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
         unique_together = ("stream", "property", "subgroup", "end_time")
         # This index dramatically improves the performance of
         # aggregating from streams to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> str:
-        return "<StreamCount: %s %s %s %s %s>" % (
-            self.stream, self.property, self.subgroup, self.value, self.id)
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ('stream_id', 'realm_id')
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Stream
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)

analytics/tests/test_fixtures.py

@@ -1,29 +0,0 @@
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-from zerver.lib.test_classes import ZulipTestCase
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self) -> None:
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=1500,
-            growth=2, spikiness=0, frequency=CountStat.HOUR)
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assertEqual(len(data), 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500*2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=2000,
-            autocorrelation=1, frequency=CountStat.HOUR)
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])

analytics/tests/test_views.py

@@ -1,355 +1,23 @@
-from datetime import datetime, timedelta
-from typing import Dict, List, Optional
-
-import mock
-from django.utils.timezone import utc
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, \
-    RealmCount, UserCount, last_successful_fill
-from analytics.views import get_chart_data, rewrite_client_arrays, \
-    sort_by_totals, sort_client_labels, stats
+from django.utils.timezone import get_fixed_timezone
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_day, \
-    ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client, get_realm
 
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        result = self.client_get('/stats')
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+from analytics.views import rewrite_client_arrays
 
-    def test_stats_for_realm(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 302)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/realm/not_existing_realm/')
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 302)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-class TestGetChartData(ZulipTestCase):
-    def setUp(self) -> None:
-        self.realm = get_realm('zulip')
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
-                               for i in range(4)]
-        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
-                              for i in range(4)]
-
-    def data(self, i: int) -> List[int]:
-        return [0, 0, i, 0]
-
-    def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]],
-                    user_subgroups: List[str]) -> None:
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create([
-            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                       value=100+i, realm=self.realm)
-            for i, subgroup in enumerate(realm_subgroups)])
-        UserCount.objects.bulk_create([
-            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                      value=200+i, realm=self.realm, user=self.user)
-            for i, subgroup in enumerate(user_subgroups)])
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_over_time(self) -> None:
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        self.insert_data(stat, ['true', 'false'], ['false'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-            'frequency': CountStat.HOUR,
-            'everyone': {'bot': self.data(100), 'human': self.data(101)},
-            'user': {'bot': self.data(0), 'human': self.data(200)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_message_type(self) -> None:
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        self.insert_data(stat, ['public_stream', 'private_message'],
-                         ['public_stream', 'private_stream'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'Public streams': self.data(100), 'Private streams': self.data(0),
-                         'Private messages': self.data(101), 'Group private messages': self.data(0)},
-            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
-                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
-            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_client(self) -> None:
-        stat = COUNT_STATS['messages_sent:client:day']
-        client1 = Client.objects.create(name='client 1')
-        client2 = Client.objects.create(name='client 2')
-        client3 = Client.objects.create(name='client 3')
-        client4 = Client.objects.create(name='client 4')
-        self.insert_data(stat, [client4.id, client3.id, client2.id],
-                         [client3.id, client1.id])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'everyone': {'client 4': self.data(100), 'client 3': self.data(101),
-                         'client 2': self.data(102)},
-            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
-            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
-            'result': 'success',
-        })
-
-    def test_include_empty_subgroups(self) -> None:
-        FillState.objects.create(
-            property='realm_active_humans::day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {"_1day": [0], "_15day": [0], "all_time": [0]})
-        self.assertFalse('user' in data)
-
-        FillState.objects.create(
-            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {'human': [0], 'bot': [0]})
-        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:message_type:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
-        self.assertEqual(data['user'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:client:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['everyone'], {})
-        self.assertEqual(data['user'], {})
-
-    def test_start_and_end(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[1],
-                                  'end': end_time_timestamps[2]})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
-        self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})
-
-        # start later then end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[2],
-                                  'end': end_time_timestamps[1]})
-        self.assert_json_error_contains(result, 'Start time is later than')
-
-    def test_min_length(self) -> None:
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
-        # test min_length is too short to change anything
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 2})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
-        self.assertEqual(data['everyone'], {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)})
-        # test min_length larger than filled data
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 5})
-        self.assert_json_success(result)
-        data = result.json()
-        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(data['everyone'], {'_1day': [0]+self.data(100), '_15day': [0]+self.data(100), 'all_time': [0]+self.data(100)})
-
-    def test_non_existent_chart(self) -> None:
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'does_not_exist'})
-        self.assert_json_error_contains(result, 'Unknown chart name')
-
-    def test_analytics_not_running(self) -> None:
-        # try to get data for a valid chart, but before we've put anything in the database
-        # (e.g. before update_analytics_counts has been run)
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'number_of_humans'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip/',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, 'Invalid organization', 400)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user_profile = self.example_user('hamlet')
-        self.login(user_profile.email)
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user_profile = self.example_user('hamlet')
-        user_profile.is_staff = True
-        user_profile.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    # last_successful_fill is in analytics/models.py, but get_chart_data is
-    # the only function that uses it at the moment
-    def test_last_successful_fill(self) -> None:
-        self.assertIsNone(last_successful_fill('non-existant'))
-        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
-        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
-        fillstate = FillState.objects.create(property='property', end_time=a_time,
-                                             state=FillState.DONE)
-        self.assertEqual(last_successful_fill('property'), a_time)
-        fillstate.state = FillState.STARTED
-        fillstate.save()
-        self.assertEqual(last_successful_fill('property'), one_hour_before)
-
-    def test_sort_by_totals(self) -> None:
-        empty = []  # type: List[int]
-        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
-        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
-
-    def test_sort_client_labels(self) -> None:
-        data = {'everyone': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
-                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
-        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
+from datetime import datetime, timedelta
 
 class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
+    def test_time_range(self):
+        # type: () -> None
         HOUR = timedelta(hours=1)
         DAY = timedelta(days=1)
+        TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC
 
-        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
-        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
-        floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
+        # Using 22:59 so that converting to UTC and applying floor_to_{hour,day} do not commute
+        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)
+        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=TZINFO)
+        floor_day = datetime(2016, 3, 14).replace(tzinfo=TZINFO)
 
         # test start == end
         self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
@@ -369,13 +37,13 @@ class TestTimeRange(ZulipTestCase):
                          [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
 
 class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
+    def test_map_arrays(self):
+        # type: () -> None
         a = {'desktop app 1.0': [1, 2, 3],
              'desktop app 2.0': [10, 12, 13],
              'desktop app 3.0': [21, 22, 23],
              'website': [1, 2, 3],
              'ZulipiOS': [1, 2, 3],
-             'ZulipElectron': [2, 5, 7],
              'ZulipMobile': [1, 5, 7],
              'ZulipPython': [1, 2, 3],
              'API: Python': [1, 2, 3],
@@ -386,10 +54,9 @@ class TestMapArrays(ZulipTestCase):
         self.assertEqual(result,
                          {'Old desktop app': [32, 36, 39],
                           'Old iOS app': [1, 2, 3],
-                          'Desktop app': [2, 5, 7],
-                          'Mobile app': [1, 5, 7],
+                          'New iOS app': [1, 5, 7],
                           'Website': [1, 2, 3],
                           'Python API': [2, 4, 6],
                           'SomethingRandom': [4, 5, 6],
                           'GitHub webhook': [7, 7, 9],
-                          'Old Android app': [64, 63, 65]})
+                          'Android app': [64, 63, 65]})

analytics/urls.py

@@ -1,7 +1,7 @@
-from django.conf.urls import include, url
+from django.conf.urls import url, include
+from zerver.lib.rest import rest_dispatch
 
 import analytics.views
-from zerver.lib.rest import rest_dispatch
 
 i18n_urlpatterns = [
     # Server admin (user_profile.is_staff) visible stats pages
@@ -12,11 +12,6 @@ i18n_urlpatterns = [
     url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
         name='analytics.views.get_user_activity'),
 
-    url(r'^stats/realm/(?P<realm_str>[\S]+)/$', analytics.views.stats_for_realm,
-        name='analytics.views.stats_for_realm'),
-    url(r'^stats/installation$', analytics.views.stats_for_installation,
-        name='analytics.views.stats_for_installation'),
-
     # User-visible stats page
     url(r'^stats$', analytics.views.stats,
         name='analytics.views.stats'),
@@ -34,10 +29,6 @@ v1_api_and_json_patterns = [
     # get data for the graphs at /stats
     url(r'^analytics/chart_data$', rest_dispatch,
         {'GET': 'analytics.views.get_chart_data'}),
-    url(r'^analytics/chart_data/realm/(?P<realm_str>[\S]+)$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_realm'}),
-    url(r'^analytics/chart_data/installation$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data_for_installation'}),
 ]
 
 i18n_urlpatterns += [

analytics/views.py

@@ -1,205 +1,103 @@
+from __future__ import absolute_import, division
 
-import itertools
-import json
-import logging
-import re
-import time
-from collections import defaultdict
-from datetime import datetime, timedelta
-from typing import Any, Callable, Dict, List, \
-    Optional, Set, Tuple, Type, Union
-
-import pytz
-from django.conf import settings
-from django.urls import reverse
+from django.core import urlresolvers
 from django.db import connection
 from django.db.models import Sum
 from django.db.models.query import QuerySet
-from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
-from django.shortcuts import render
+from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from django.template import RequestContext, loader
-from django.utils.timezone import now as timezone_now, utc as timezone_utc
+from django.utils import timezone
 from django.utils.translation import ugettext as _
 from jinja2 import Markup as mark_safe
 
-from analytics.lib.counts import COUNT_STATS, CountStat, process_count_stat
+from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS
 from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, InstallationCount, \
-    RealmCount, StreamCount, UserCount, last_successful_fill, installation_epoch
-from zerver.decorator import require_server_admin, require_server_admin_api, \
-    to_non_negative_int, to_utc_datetime, zulip_login_required
-from zerver.lib.exceptions import JsonableError
-from zerver.lib.json_encoder_for_html import JSONEncoderForHTML
-from zerver.lib.request import REQ, has_request_variables
-from zerver.lib.response import json_success
-from zerver.lib.timestamp import ceiling_to_day, \
-    ceiling_to_hour, convert_to_UTC, timestamp_to_datetime
-from zerver.models import Client, get_realm, Realm, \
-    UserActivity, UserActivityInterval, UserProfile
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount
 
-def render_stats(request: HttpRequest, data_url_suffix: str, target_name: str,
-                 for_installation: bool=False) -> HttpRequest:
-    page_params = dict(
-        data_url_suffix=data_url_suffix,
-        for_installation=for_installation,
-        debug_mode=False,
-    )
-    return render(request,
-                  'analytics/stats.html',
-                  context=dict(target_name=target_name,
-                               page_params=JSONEncoderForHTML().encode(page_params)))
+from zerver.decorator import has_request_variables, REQ, zulip_internal, \
+    zulip_login_required, to_non_negative_int, to_utc_datetime
+from zerver.lib.request import JsonableError
+from zerver.lib.response import json_success
+from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, timestamp_to_datetime
+from zerver.models import Realm, UserProfile, UserActivity, \
+    UserActivityInterval, Client
+from zproject.jinja2 import render_to_response
+
+from collections import defaultdict
+from datetime import datetime, timedelta
+import itertools
+import json
+import pytz
+import re
+import time
+
+from six.moves import filter, map, range, zip
+from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Type, \
+    Union, Text
 
 @zulip_login_required
-def stats(request: HttpRequest) -> HttpResponse:
-    realm = request.user.realm
-    return render_stats(request, '', realm.name or realm.string_id)
-
-@require_server_admin
-@has_request_variables
-def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
-    realm = get_realm(realm_str)
-    if realm is None:
-        return HttpResponseNotFound("Realm %s does not exist" % (realm_str,))
-
-    return render_stats(request, '/realm/%s' % (realm_str,), realm.name or realm.string_id)
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_realm(request: HttpRequest, user_profile: UserProfile,
-                             realm_str: str, **kwargs: Any) -> HttpResponse:
-    realm = get_realm(realm_str)
-    if realm is None:
-        raise JsonableError(_("Invalid organization"))
-
-    return get_chart_data(request=request, user_profile=user_profile, realm=realm, **kwargs)
-
-@require_server_admin
-def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    return render_stats(request, '/installation', 'Installation', True)
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_installation(request: HttpRequest, user_profile: UserProfile,
-                                    chart_name: str=REQ(), **kwargs: Any) -> HttpResponse:
-    return get_chart_data(request=request, user_profile=user_profile, for_installation=True, **kwargs)
+def stats(request):
+    # type: (HttpRequest) -> HttpResponse
+    return render_to_response('analytics/stats.html',
+                              context=dict(realm_name = request.user.realm.name))
 
 @has_request_variables
-def get_chart_data(request: HttpRequest, user_profile: UserProfile, chart_name: str=REQ(),
-                   min_length: Optional[int]=REQ(converter=to_non_negative_int, default=None),
-                   start: Optional[datetime]=REQ(converter=to_utc_datetime, default=None),
-                   end: Optional[datetime]=REQ(converter=to_utc_datetime, default=None),
-                   realm: Optional[Realm]=None, for_installation: bool=False) -> HttpResponse:
-    aggregate_table = RealmCount
-    if for_installation:
-        aggregate_table = InstallationCount
+def get_chart_data(request, user_profile, chart_name=REQ(),
+                   min_length=REQ(converter=to_non_negative_int, default=None),
+                   start=REQ(converter=to_utc_datetime, default=None),
+                   end=REQ(converter=to_utc_datetime, default=None)):
+    # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse
+    realm = user_profile.realm
+    # These are implicitly relying on realm.date_created and timezone.now being in UTC.
+    if start is None:
+        start = realm.date_created
+    if end is None:
+        end = timezone.now()
+    if start > end:
+        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
+                            {'start': start, 'end': end})
 
     if chart_name == 'number_of_humans':
-        stats = [
-            COUNT_STATS['1day_actives::day'],
-            COUNT_STATS['realm_active_humans::day'],
-            COUNT_STATS['active_users_audit:is_bot:day']]
-        tables = [aggregate_table]
-        subgroup_to_label = {
-            stats[0]: {None: '_1day'},
-            stats[1]: {None: '_15day'},
-            stats[2]: {'false': 'all_time'}}  # type: Dict[CountStat, Dict[Optional[str], str]]
-        labels_sort_function = None
+        stat = COUNT_STATS['active_users:is_bot:day']
+        tables = [RealmCount]
+        subgroups = ['false', 'true']
+        labels = ['human', 'bot']
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_over_time':
-        stats = [COUNT_STATS['messages_sent:is_bot:hour']]
-        tables = [aggregate_table, UserCount]
-        subgroup_to_label = {stats[0]: {'false': 'human', 'true': 'bot'}}
-        labels_sort_function = None
+        stat = COUNT_STATS['messages_sent:is_bot:hour']
+        tables = [RealmCount]
+        subgroups = ['false', 'true']
+        labels = ['human', 'bot']
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_by_message_type':
-        stats = [COUNT_STATS['messages_sent:message_type:day']]
-        tables = [aggregate_table, UserCount]
-        subgroup_to_label = {stats[0]: {'public_stream': _('Public streams'),
-                                        'private_stream': _('Private streams'),
-                                        'private_message': _('Private messages'),
-                                        'huddle_message': _('Group private messages')}}
-        labels_sort_function = lambda data: sort_by_totals(data['everyone'])
+        stat = COUNT_STATS['messages_sent:message_type:day']
+        tables = [RealmCount, UserCount]
+        subgroups = ['public_stream', 'private_stream', 'private_message']
+        labels = None
         include_empty_subgroups = True
     elif chart_name == 'messages_sent_by_client':
-        stats = [COUNT_STATS['messages_sent:client:day']]
-        tables = [aggregate_table, UserCount]
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {stats[0]:
-                             {str(id): name for id, name in Client.objects.values_list('id', 'name')}}
-        labels_sort_function = sort_client_labels
+        stat = COUNT_STATS['messages_sent:client:day']
+        tables = [RealmCount, UserCount]
+        subgroups = [str(x) for x in Client.objects.values_list('id', flat=True).order_by('id')]
+        labels = list(Client.objects.values_list('name', flat=True).order_by('id'))
         include_empty_subgroups = False
     else:
         raise JsonableError(_("Unknown chart name: %s") % (chart_name,))
 
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None:
-        start = convert_to_UTC(start)
-    if end is not None:
-        end = convert_to_UTC(end)
-    if start is not None and end is not None and start > end:
-        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
-                            {'start': start, 'end': end})
-
-    if realm is None:
-        realm = user_profile.realm
-    if start is None:
-        if for_installation:
-            start = installation_epoch()
-        else:
-            start = realm.date_created
-    if end is None:
-        end = max(last_successful_fill(stat.property) or
-                  datetime.min.replace(tzinfo=timezone_utc) for stat in stats)
-    if end is None or start > end:
-        logging.warning("User from realm %s attempted to access /stats, but the computed "
-                        "start time: %s (creation of realm or installation) is later than the computed "
-                        "end time: %s (last successful analytics update). Is the "
-                        "analytics cron job running?" % (realm.string_id, start, end))
-        raise JsonableError(_("No analytics data available. Please contact your server administrator."))
-
-    assert len(set([stat.frequency for stat in stats])) == 1
-    end_times = time_range(start, end, stats[0].frequency, min_length)
-    data = {'end_times': end_times, 'frequency': stats[0].frequency}  # type: Dict[str, Any]
-
-    aggregation_level = {InstallationCount: 'everyone', RealmCount: 'everyone', UserCount: 'user'}
-    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
-    id_value = {InstallationCount: -1, RealmCount: realm.id, UserCount: user_profile.id}
+    end_times = time_range(start, end, stat.frequency, min_length)
+    data = {'end_times': end_times, 'frequency': stat.frequency, 'interval': stat.interval}
     for table in tables:
-        data[aggregation_level[table]] = {}
-        for stat in stats:
-            data[aggregation_level[table]].update(get_time_series_by_subgroup(
-                stat, table, id_value[table], end_times, subgroup_to_label[stat], include_empty_subgroups))
-
-    if labels_sort_function is not None:
-        data['display_order'] = labels_sort_function(data)
-    else:
-        data['display_order'] = None
+        if table == RealmCount:
+            data['realm'] = get_time_series_by_subgroup(
+                stat, RealmCount, realm.id, end_times, subgroups, labels, include_empty_subgroups)
+        if table == UserCount:
+            data['user'] = get_time_series_by_subgroup(
+                stat, UserCount, user_profile.id, end_times, subgroups, labels, include_empty_subgroups)
     return json_success(data=data)
 
-def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
-    totals = [(sum(values), label) for label, values in value_arrays.items()]
-    totals.sort(reverse=True)
-    return [label for total, label in totals]
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data: Dict[str, Dict[str, List[int]]]) -> List[str]:
-    realm_order = sort_by_totals(data['everyone'])
-    user_order = sort_by_totals(data['user'])
-    label_sort_values = {}  # type: Dict[str, float]
-    for i, label in enumerate(realm_order):
-        label_sort_values[label] = i
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i-.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(),
-                                                  key=lambda x: x[1])]
-
-def table_filtered_to_id(table: Type[BaseCount], key_id: int) -> QuerySet:
+def table_filtered_to_id(table, key_id):
+    # type: (Type[BaseCount], int) -> QuerySet
     if table == RealmCount:
         return RealmCount.objects.filter(realm_id=key_id)
     elif table == UserCount:
@@ -209,29 +107,29 @@ def table_filtered_to_id(table: Type[BaseCount], key_id: int) -> QuerySet:
     elif table == InstallationCount:
         return InstallationCount.objects.all()
     else:
-        raise AssertionError("Unknown table: %s" % (table,))
+        raise ValueError("Unknown table: %s" % (table,))
 
-def client_label_map(name: str) -> str:
+def client_label_map(name):
+    # type: (str) -> str
     if name == "website":
         return "Website"
     if name.startswith("desktop app"):
         return "Old desktop app"
-    if name == "ZulipElectron":
-        return "Desktop app"
     if name == "ZulipAndroid":
-        return "Old Android app"
+        return "Android app"
     if name == "ZulipiOS":
         return "Old iOS app"
     if name == "ZulipMobile":
-        return "Mobile app"
+        return "New iOS app"
     if name in ["ZulipPython", "API: Python"]:
         return "Python API"
     if name.startswith("Zulip") and name.endswith("Webhook"):
         return name[len("Zulip"):-len("Webhook")] + " webhook"
     return name
 
-def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[int]]:
-    mapped_arrays = {}  # type: Dict[str, List[int]]
+def rewrite_client_arrays(value_arrays):
+    # type: (Dict[str, List[int]]) -> Dict[str, List[int]]
+    mapped_arrays = {} # type: Dict[str, List[int]]
     for label, array in value_arrays.items():
         mapped_label = client_label_map(label)
         if mapped_label in mapped_arrays:
@@ -241,19 +139,20 @@ def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[
             mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
     return mapped_arrays
 
-def get_time_series_by_subgroup(stat: CountStat,
-                                table: Type[BaseCount],
-                                key_id: int,
-                                end_times: List[datetime],
-                                subgroup_to_label: Dict[Optional[str], str],
-                                include_empty_subgroups: bool) -> Dict[str, List[int]]:
+def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroups, labels, include_empty_subgroups):
+    # type: (CountStat, Type[BaseCount], Optional[int], List[datetime], List[str], Optional[List[str]], bool) -> Dict[str, List[int]]
+    if labels is None:
+        labels = subgroups
+    if len(subgroups) != len(labels):
+        raise ValueError("subgroups and labels have lengths %s and %s, which are different." %
+                         (len(subgroups), len(labels)))
     queryset = table_filtered_to_id(table, key_id).filter(property=stat.property) \
                                                   .values_list('subgroup', 'end_time', 'value')
-    value_dicts = defaultdict(lambda: defaultdict(int))  # type: Dict[Optional[str], Dict[datetime, int]]
+    value_dicts = defaultdict(lambda: defaultdict(int)) # type: Dict[Optional[str], Dict[datetime, int]]
     for subgroup, end_time, value in queryset:
         value_dicts[subgroup][end_time] = value
     value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
+    for subgroup, label in zip(subgroups, labels):
         if (subgroup in value_dicts) or include_empty_subgroups:
             value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
 
@@ -267,10 +166,12 @@ def get_time_series_by_subgroup(stat: CountStat,
 
 eastern_tz = pytz.timezone('US/Eastern')
 
-def make_table(title: str, cols: List[str], rows: List[Any], has_row_class: bool=False) -> str:
+def make_table(title, cols, rows, has_row_class=False):
+    # type: (str, List[str], List[Any], bool) -> str
 
     if not has_row_class:
-        def fix_row(row: Any) -> Dict[str, Any]:
+        def fix_row(row):
+            # type: (Any) -> Dict[str, Any]
             return dict(cells=row, row_class=None)
         rows = list(map(fix_row, rows))
 
@@ -283,7 +184,8 @@ def make_table(title: str, cols: List[str], rows: List[Any], has_row_class: bool
 
     return content
 
-def dictfetchall(cursor: connection.cursor) -> List[Dict[str, Any]]:
+def dictfetchall(cursor):
+    # type: (connection.cursor) -> List[Dict[str, Any]]
     "Returns all rows from a cursor as a dict"
     desc = cursor.description
     return [
@@ -292,7 +194,8 @@ def dictfetchall(cursor: connection.cursor) -> List[Dict[str, Any]]:
     ]
 
 
-def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
+def get_realm_day_counts():
+    # type: () -> Dict[str, Dict[str, str]]
     query = '''
         select
             r.string_id,
@@ -320,20 +223,19 @@ def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
     rows = dictfetchall(cursor)
     cursor.close()
 
-    counts = defaultdict(dict)  # type: Dict[str, Dict[int, int]]
+    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
     for row in rows:
         counts[row['string_id']][row['age']] = row['cnt']
 
     result = {}
     for string_id in counts:
         raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
-        min_cnt = min(raw_cnts[1:])
-        max_cnt = max(raw_cnts[1:])
+        min_cnt = min(raw_cnts)
+        max_cnt = max(raw_cnts)
 
-        def format_count(cnt: int, style: Optional[str]=None) -> str:
-            if style is not None:
-                good_bad = style
-            elif cnt == min_cnt:
+        def format_count(cnt):
+            # type: (int) -> str
+            if cnt == min_cnt:
                 good_bad = 'bad'
             elif cnt == max_cnt:
                 good_bad = 'good'
@@ -342,22 +244,18 @@ def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
 
             return '<td class="number %s">%s</td>' % (good_bad, cnt)
 
-        cnts = (format_count(raw_cnts[0], 'neutral')
-                + ''.join(map(format_count, raw_cnts[1:])))
+        cnts = ''.join(map(format_count, raw_cnts))
         result[string_id] = dict(cnts=cnts)
 
     return result
 
-def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
-    now = timezone_now()
-
+def realm_summary_table(realm_minutes):
+    # type: (Dict[str, float]) -> str
     query = '''
         SELECT
             realm.string_id,
-            realm.date_created,
-            realm.plan_type,
-            coalesce(user_counts.dau_count, 0) dau_count,
-            coalesce(wau_counts.wau_count, 0) wau_count,
+            coalesce(user_counts.active_user_count, 0) active_user_count,
+            coalesce(at_risk_counts.at_risk_count, 0) at_risk_count,
             (
                 SELECT
                     count(*)
@@ -379,24 +277,22 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
             (
                 SELECT
                     up.realm_id realm_id,
-                    count(distinct(ua.user_profile_id)) dau_count
+                    count(distinct(ua.user_profile_id)) active_user_count
                 FROM zerver_useractivity ua
                 JOIN zerver_userprofile up
                     ON up.id = ua.user_profile_id
                 WHERE
-                    up.is_active
-                AND (not up.is_bot)
-                AND
                     query in (
                         '/json/send_message',
                         'send_message_backend',
                         '/api/v1/send_message',
                         '/json/update_pointer',
-                        '/json/users/me/pointer',
-                        'update_pointer_backend'
+                        '/json/users/me/pointer'
                     )
                 AND
                     last_visit > now() - interval '1 day'
+                AND
+                    not is_bot
                 GROUP BY realm_id
             ) user_counts
             ON user_counts.realm_id = realm.id
@@ -404,7 +300,7 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
             (
                 SELECT
                     realm_id,
-                    count(*) wau_count
+                    count(*) at_risk_count
                 FROM (
                     SELECT
                         realm.id as realm_id,
@@ -422,37 +318,35 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
                             'send_message_backend',
                             '/api/v1/send_message',
                             '/json/update_pointer',
-                            '/json/users/me/pointer',
-                            'update_pointer_backend'
+                            '/json/users/me/pointer'
                         )
                     GROUP by realm.id, up.email
-                    HAVING max(last_visit) > now() - interval '7 day'
-                ) as wau_users
+                    HAVING max(last_visit) between
+                        now() - interval '7 day' and
+                        now() - interval '1 day'
+                ) as at_risk_users
                 GROUP BY realm_id
-            ) wau_counts
-            ON wau_counts.realm_id = realm.id
+            ) at_risk_counts
+            ON at_risk_counts.realm_id = realm.id
         WHERE EXISTS (
                 SELECT *
                 FROM zerver_useractivity ua
                 JOIN zerver_userprofile up
                     ON up.id = ua.user_profile_id
                 WHERE
-                    up.realm_id = realm.id
-                AND up.is_active
-                AND (not up.is_bot)
-                AND
                     query in (
                         '/json/send_message',
                         '/api/v1/send_message',
                         'send_message_backend',
                         '/json/update_pointer',
-                        '/json/users/me/pointer',
-                        'update_pointer_backend'
+                        '/json/users/me/pointer'
                     )
+                AND
+                    up.realm_id = realm.id
                 AND
                     last_visit > now() - interval '2 week'
         )
-        ORDER BY dau_count DESC, string_id ASC
+        ORDER BY active_user_count DESC, string_id ASC
         '''
 
     cursor = connection.cursor()
@@ -460,23 +354,6 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
     rows = dictfetchall(cursor)
     cursor.close()
 
-    # Fetch all the realm administrator users
-    realm_admins = defaultdict(list)  # type: Dict[str, List[str]]
-    for up in UserProfile.objects.select_related("realm").filter(
-        is_realm_admin=True,
-        is_active=True
-    ):
-        realm_admins[up.realm.string_id].append(up.email)
-
-    for row in rows:
-        row['date_created_day'] = row['date_created'].strftime('%Y-%m-%d')
-        row['plan_type_string'] = [
-            '', 'self hosted', 'limited', 'premium', 'premium free'][row['plan_type']]
-        row['age_days'] = int((now - row['date_created']).total_seconds()
-                              / 86400)
-        row['is_new'] = row['age_days'] < 12 * 7
-        row['realm_admin_email'] = ', '.join(realm_admins[row['string_id']])
-
     # get messages sent per day
     counts = get_realm_day_counts()
     for row in rows:
@@ -494,54 +371,50 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
         total_hours += hours
         row['hours'] = str(int(hours))
         try:
-            row['hours_per_user'] = '%.1f' % (hours / row['dau_count'],)
+            row['hours_per_user'] = '%.1f' % (hours / row['active_user_count'],)
         except Exception:
             pass
 
     # formatting
     for row in rows:
-        row['stats_link'] = realm_stats_link(row['string_id'])
         row['string_id'] = realm_activity_link(row['string_id'])
 
     # Count active sites
-    def meets_goal(row: Dict[str, int]) -> bool:
-        return row['dau_count'] >= 5
+    def meets_goal(row):
+        # type: (Dict[str, int]) -> bool
+        return row['active_user_count'] >= 5
 
     num_active_sites = len(list(filter(meets_goal, rows)))
 
     # create totals
-    total_dau_count = 0
+    total_active_user_count = 0
     total_user_profile_count = 0
     total_bot_count = 0
-    total_wau_count = 0
+    total_at_risk_count = 0
     for row in rows:
-        total_dau_count += int(row['dau_count'])
+        total_active_user_count += int(row['active_user_count'])
         total_user_profile_count += int(row['user_profile_count'])
         total_bot_count += int(row['bot_count'])
-        total_wau_count += int(row['wau_count'])
+        total_at_risk_count += int(row['at_risk_count'])
 
     rows.append(dict(
         string_id='Total',
-        plan_type_string="",
-        stats_link = '',
-        date_created_day='',
-        realm_admin_email='',
-        dau_count=total_dau_count,
+        active_user_count=total_active_user_count,
         user_profile_count=total_user_profile_count,
         bot_count=total_bot_count,
         hours=int(total_hours),
-        wau_count=total_wau_count,
+        at_risk_count=total_at_risk_count,
     ))
 
     content = loader.render_to_string(
         'analytics/realm_summary_table.html',
-        dict(rows=rows, num_active_sites=num_active_sites,
-             now=now.strftime('%Y-%m-%dT%H:%M:%SZ'))
+        dict(rows=rows, num_active_sites=num_active_sites)
     )
     return content
 
 
-def user_activity_intervals() -> Tuple[mark_safe, Dict[str, float]]:
+def user_activity_intervals():
+    # type: () -> Tuple[mark_safe, Dict[str, float]]
     day_end = timestamp_to_datetime(time.time())
     day_start = day_end - timedelta(hours=24)
 
@@ -591,7 +464,8 @@ def user_activity_intervals() -> Tuple[mark_safe, Dict[str, float]]:
     content = mark_safe('<pre>' + output + '</pre>')
     return content, realm_minutes
 
-def sent_messages_report(realm: str) -> str:
+def sent_messages_report(realm):
+    # type: (str) -> str
     title = 'Recently sent messages for ' + realm
 
     cols = [
@@ -658,16 +532,18 @@ def sent_messages_report(realm: str) -> str:
 
     return make_table(title, cols, rows)
 
-def ad_hoc_queries() -> List[Dict[str, str]]:
-    def get_page(query: str, cols: List[str], title: str) -> Dict[str, str]:
+def ad_hoc_queries():
+    # type: () -> List[Dict[str, str]]
+    def get_page(query, cols, title):
+        # type: (str, List[str], str) -> Dict[str, str]
         cursor = connection.cursor()
         cursor.execute(query)
         rows = cursor.fetchall()
         rows = list(map(list, rows))
         cursor.close()
 
-        def fix_rows(i: int,
-                     fixup_func: Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None:
+        def fix_rows(i, fixup_func):
+            # type: (int, Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None
             for row in rows:
                 row[i] = fixup_func(row[i])
 
@@ -827,11 +703,12 @@ def ad_hoc_queries() -> List[Dict[str, str]]:
 
     return pages
 
-@require_server_admin
+@zulip_internal
 @has_request_variables
-def get_activity(request: HttpRequest) -> HttpResponse:
-    duration_content, realm_minutes = user_activity_intervals()  # type: Tuple[mark_safe, Dict[str, float]]
-    counts_content = realm_summary_table(realm_minutes)  # type: str
+def get_activity(request):
+    # type: (HttpRequest) -> HttpResponse
+    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
+    counts_content = realm_summary_table(realm_minutes) # type: str
     data = [
         ('Counts', counts_content),
         ('Durations', duration_content),
@@ -841,13 +718,14 @@ def get_activity(request: HttpRequest) -> HttpResponse:
 
     title = 'Activity'
 
-    return render(
-        request,
+    return render_to_response(
         'analytics/activity.html',
-        context=dict(data=data, title=title, is_home=True),
+        dict(data=data, title=title, is_home=True),
+        request=request
     )
 
-def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet:
+def get_user_activity_records_for_realm(realm, is_bot):
+    # type: (str, bool) -> QuerySet
     fields = [
         'user_profile__full_name',
         'user_profile__email',
@@ -866,7 +744,8 @@ def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet:
     records = records.select_related('user_profile', 'client').only(*fields)
     return records
 
-def get_user_activity_records_for_email(email: str) -> List[QuerySet]:
+def get_user_activity_records_for_email(email):
+    # type: (str) -> List[QuerySet]
     fields = [
         'user_profile__full_name',
         'query',
@@ -882,7 +761,8 @@ def get_user_activity_records_for_email(email: str) -> List[QuerySet]:
     records = records.select_related('user_profile', 'client').only(*fields)
     return records
 
-def raw_user_activity_table(records: List[QuerySet]) -> str:
+def raw_user_activity_table(records):
+    # type: (List[QuerySet]) -> str
     cols = [
         'query',
         'client',
@@ -890,7 +770,8 @@ def raw_user_activity_table(records: List[QuerySet]) -> str:
         'last_visit'
     ]
 
-    def row(record: QuerySet) -> List[Any]:
+    def row(record):
+        # type: (QuerySet) -> List[Any]
         return [
             record.query,
             record.client.name,
@@ -902,15 +783,17 @@ def raw_user_activity_table(records: List[QuerySet]) -> str:
     title = 'Raw Data'
     return make_table(title, cols, rows)
 
-def get_user_activity_summary(records: List[QuerySet]) -> Dict[str, Dict[str, Any]]:
+def get_user_activity_summary(records):
+    # type: (List[QuerySet]) -> Dict[str, Dict[str, Any]]
     #: `Any` used above should be `Union(int, datetime)`.
     #: However current version of `Union` does not work inside other function.
     #: We could use something like:
     # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
     #: but that would require this long `Union` to carry on throughout inner functions.
-    summary = {}  # type: Dict[str, Dict[str, Any]]
+    summary = {} # type: Dict[str, Dict[str, Any]]
 
-    def update(action: str, record: QuerySet) -> None:
+    def update(action, record):
+        # type: (str, QuerySet) -> None
         if action not in summary:
             summary[action] = dict(
                 count=record.count,
@@ -944,38 +827,35 @@ def get_user_activity_summary(records: List[QuerySet]) -> Dict[str, Dict[str, An
             update('website', record)
         if ('send_message' in query) or re.search('/api/.*/external/.*', query):
             update('send', record)
-        if query in ['/json/update_pointer', '/json/users/me/pointer', '/api/v1/update_pointer',
-                     'update_pointer_backend']:
+        if query in ['/json/update_pointer', '/json/users/me/pointer', '/api/v1/update_pointer']:
             update('pointer', record)
         update(client, record)
 
     return summary
 
-def format_date_for_activity_reports(date: Optional[datetime]) -> str:
+def format_date_for_activity_reports(date):
+    # type: (Optional[datetime]) -> str
     if date:
         return date.astimezone(eastern_tz).strftime('%Y-%m-%d %H:%M')
     else:
         return ''
 
-def user_activity_link(email: str) -> mark_safe:
+def user_activity_link(email):
+    # type: (str) -> mark_safe
     url_name = 'analytics.views.get_user_activity'
-    url = reverse(url_name, kwargs=dict(email=email))
+    url = urlresolvers.reverse(url_name, kwargs=dict(email=email))
     email_link = '<a href="%s">%s</a>' % (url, email)
     return mark_safe(email_link)
 
-def realm_activity_link(realm_str: str) -> mark_safe:
+def realm_activity_link(realm_str):
+    # type: (str) -> mark_safe
     url_name = 'analytics.views.get_realm_activity'
-    url = reverse(url_name, kwargs=dict(realm_str=realm_str))
+    url = urlresolvers.reverse(url_name, kwargs=dict(realm_str=realm_str))
     realm_link = '<a href="%s">%s</a>' % (url, realm_str)
     return mark_safe(realm_link)
 
-def realm_stats_link(realm_str: str) -> mark_safe:
-    url_name = 'analytics.views.stats_for_realm'
-    url = reverse(url_name, kwargs=dict(realm_str=realm_str))
-    stats_link = '<a href="{}"><i class="fa fa-pie-chart"></i></a>'.format(url, realm_str)
-    return mark_safe(stats_link)
-
-def realm_client_table(user_summaries: Dict[str, Dict[str, Dict[str, Any]]]) -> str:
+def realm_client_table(user_summaries):
+    # type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
     exclude_keys = [
         'internal',
         'name',
@@ -1019,7 +899,8 @@ def realm_client_table(user_summaries: Dict[str, Dict[str, Dict[str, Any]]]) ->
 
     return make_table(title, cols, rows)
 
-def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
+def user_activity_summary_table(user_summary):
+    # type: (Dict[str, Dict[str, Any]]) -> str
     rows = []
     for k, v in user_summary.items():
         if k == 'name':
@@ -1045,30 +926,34 @@ def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
     title = 'User Activity'
     return make_table(title, cols, rows)
 
-def realm_user_summary_table(all_records: List[QuerySet],
-                             admin_emails: Set[str]) -> Tuple[Dict[str, Dict[str, Any]], str]:
+def realm_user_summary_table(all_records, admin_emails):
+    # type: (List[QuerySet], Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]
     user_records = {}
 
-    def by_email(record: QuerySet) -> str:
+    def by_email(record):
+        # type: (QuerySet) -> str
         return record.user_profile.email
 
     for email, records in itertools.groupby(all_records, by_email):
         user_records[email] = get_user_activity_summary(list(records))
 
-    def get_last_visit(user_summary: Dict[str, Dict[str, datetime]], k: str) -> Optional[datetime]:
+    def get_last_visit(user_summary, k):
+        # type: (Dict[str, Dict[str, datetime]], str) -> Optional[datetime]
         if k in user_summary:
             return user_summary[k]['last_visit']
         else:
             return None
 
-    def get_count(user_summary: Dict[str, Dict[str, str]], k: str) -> str:
+    def get_count(user_summary, k):
+        # type: (Dict[str, Dict[str, str]], str) -> str
         if k in user_summary:
             return user_summary[k]['count']
         else:
             return ''
 
-    def is_recent(val: Optional[datetime]) -> bool:
-        age = timezone_now() - val
+    def is_recent(val):
+        # type: (Optional[datetime]) -> bool
+        age = datetime.now(val.tzinfo) - val
         return age.total_seconds() < 5 * 60
 
     rows = []
@@ -1089,7 +974,8 @@ def realm_user_summary_table(all_records: List[QuerySet],
         row = dict(cells=cells, row_class=row_class)
         rows.append(row)
 
-    def by_used_time(row: Dict[str, Any]) -> str:
+    def by_used_time(row):
+        # type: (Dict[str, Sequence[str]]) -> str
         return row['cells'][3]
 
     rows = sorted(rows, key=by_used_time, reverse=True)
@@ -1111,10 +997,11 @@ def realm_user_summary_table(all_records: List[QuerySet],
     content = make_table(title, cols, rows, has_row_class=True)
     return user_records, content
 
-@require_server_admin
-def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
-    data = []  # type: List[Tuple[str, str]]
-    all_user_records = {}  # type: Dict[str, Any]
+@zulip_internal
+def get_realm_activity(request, realm_str):
+    # type: (HttpRequest, str) -> HttpResponse
+    data = [] # type: List[Tuple[str, str]]
+    all_user_records = {} # type: Dict[str, Any]
 
     try:
         admins = Realm.objects.get(string_id=realm_str).get_admin_users()
@@ -1139,18 +1026,22 @@ def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
     content = sent_messages_report(realm_str)
     data += [(page_title, content)]
 
+    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
+    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (realm_str,)
+
     title = realm_str
-    return render(
-        request,
+    return render_to_response(
         'analytics/activity.html',
-        context=dict(data=data, realm_link=None, title=title),
+        dict(data=data, realm_link=realm_link, title=title),
+        request=request
     )
 
-@require_server_admin
-def get_user_activity(request: HttpRequest, email: str) -> HttpResponse:
+@zulip_internal
+def get_user_activity(request, email):
+    # type: (HttpRequest, str) -> HttpResponse
     records = get_user_activity_records_for_email(email)
 
-    data = []  # type: List[Tuple[str, str]]
+    data = [] # type: List[Tuple[str, str]]
     user_summary = get_user_activity_summary(records)
     content = user_activity_summary_table(user_summary)
 
@@ -1160,8 +1051,8 @@ def get_user_activity(request: HttpRequest, email: str) -> HttpResponse:
     data += [('Info', content)]
 
     title = email
-    return render(
-        request,
+    return render_to_response(
         'analytics/activity.html',
-        context=dict(data=data, title=title),
+        dict(data=data, title=title),
+        request=request
     )

api/MANIFEST.in

@@ -0,0 +1,11 @@
+recursive-include integrations *
+include README.md
+include examples/zuliprc
+include examples/send-message
+include examples/subscribe
+include examples/get-public-streams
+include examples/unsubscribe
+include examples/list-members
+include examples/list-subscriptions
+include examples/print-messages
+include examples/recent-messages

api/README.md

@@ -0,0 +1,172 @@
+#### Dependencies
+
+The [Zulip API](https://zulipchat.com/api) Python bindings require the
+following Python libraries:
+
+* requests (version >= 0.12.1)
+* simplejson
+* six
+* typing (version >= 3.5.2.2)
+
+#### Installing
+
+This package uses distutils, so you can just run:
+
+    python setup.py install
+
+#### Using the API
+
+For now, the only fully supported API operation is sending a message.
+The other API queries work, but are under active development, so
+please make sure we know you're using them so that we can notify you
+as we make any changes to them.
+
+The easiest way to use these API bindings is to base your tools off
+of the example tools under examples/ in this distribution.
+
+If you place your API key in the config file `~/.zuliprc` the Python
+API bindings will automatically read it in. The format of the config
+file is as follows:
+
+    [api]
+    key=<api key from the web interface>
+    email=<your email address>
+    site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
+
+Alternatively, you may explicitly use "--user", "--api-key", and
+`--site` in our examples, which is especially useful when testing.  If
+you are running several bots which share a home directory, we
+recommend using `--config` to specify the path to the `zuliprc` file
+for a specific bot.
+
+The command line equivalents for other configuration options are:
+
+    --insecure
+    --cert-bundle=<file>
+
+You can obtain your Zulip API key, create bots, and manage bots all
+from your Zulip settings page; with current Zulip there's also a
+button to download a `zuliprc` file for your account/server pair.
+
+A typical simple bot sending API messages will look as follows:
+
+At the top of the file:
+
+    # Make sure the Zulip API distribution's root directory is in sys.path, then:
+    import zulip
+    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
+
+When you want to send a message:
+
+    message = {
+      "type": "stream",
+      "to": ["support"],
+      "subject": "your subject",
+      "content": "your content",
+    }
+    zulip_client.send_message(message)
+
+If you are parsing arguments, you may find it useful to use Zulip's
+option group; see any of our API examples for details on how to do this.
+
+Additional examples:
+
+    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
+                         'subject': 'feedback', 'to': ['support']})
+    client.send_message({'type': 'private', 'content': 'Zulip rules!',
+                         'to': ['user1@example.com', 'user2@example.com']})
+
+send_message() returns a dict guaranteed to contain the following
+keys: msg, result.  For successful calls, result will be "success" and
+msg will be the empty string.  On error, result will be "error" and
+msg will describe what went wrong.
+
+#### Examples
+
+The API bindings package comes with several nice example scripts that
+show how to use the APIs; they are installed as part of the API
+bindings bundle.
+
+#### Logging
+
+The Zulip API comes with a ZulipStream class which can be used with the
+logging module:
+
+```
+import zulip
+import logging
+stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
+logger = logging.getLogger("your_logger")
+logger.addHandler(logging.StreamHandler(stream))
+logger.setLevel(logging.DEBUG)
+logger.info("This is an INFO test.")
+logger.debug("This is a DEBUG test.")
+logger.warn("This is a WARN test.")
+logger.error("This is a ERROR test.")
+```
+
+#### Sending messages
+
+You can use the included `zulip-send` script to send messages via the
+API directly from existing scripts.
+
+    zulip-send hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+Alternatively, if you don't want to use your ~/.zuliprc file:
+
+    zulip-send --user shakespeare-bot@example.com \
+        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+        --site https://zulip.example.com \
+        hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.

api/bin/zulip-send

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# zulip-send -- Sends a message to the specified recipients.
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import os
+import optparse
+import logging
+
+from typing import Any, Dict, List, Optional
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+import zulip
+
+logging.basicConfig()
+
+log = logging.getLogger('zulip-send')
+
+def do_send_message(client, message_data):
+    # type: (zulip.Client, Dict[str, Any]) -> bool
+    '''Sends a message and optionally prints status about the same.'''
+
+    if message_data['type'] == 'stream':
+        log.info('Sending message to stream "%s", subject "%s"... ' %
+                 (message_data['to'], message_data['subject']))
+    else:
+        log.info('Sending message to %s... ' % message_data['to'])
+    response = client.send_message(message_data)
+    if response['result'] == 'success':
+        log.info('Message sent.')
+        return True
+    else:
+        log.error(response['msg'])
+        return False
+
+def main(argv=None):
+    # type: (Optional[List[str]]) -> int
+    if argv is None:
+        argv = sys.argv
+
+    usage = """%prog [options] [recipient...]
+
+    Sends a message specified recipients.
+
+    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
+              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
+
+    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
+    '--user' and '--api-key' arguments.
+    """
+
+    parser = optparse.OptionParser(usage=usage)
+
+    # Grab parser options from the API common set
+    parser.add_option_group(zulip.generate_option_group(parser))
+
+    parser.add_option('-m', '--message',
+                      help='Specifies the message to send, prevents interactive prompting.')
+
+    group = optparse.OptionGroup(parser, 'Stream parameters')
+    group.add_option('-s', '--stream',
+                     dest='stream',
+                     action='store',
+                     help='Allows the user to specify a stream for the message.')
+    group.add_option('-S', '--subject',
+                     dest='subject',
+                     action='store',
+                     help='Allows the user to specify a subject for the message.')
+    parser.add_option_group(group)
+
+    (options, recipients) = parser.parse_args(argv[1:])
+
+    if options.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+    # Sanity check user data
+    if len(recipients) != 0 and (options.stream or options.subject):
+        parser.error('You cannot specify both a username and a stream/subject.')
+    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
+        parser.error('Stream messages must have a subject')
+    if len(recipients) == 0 and not (options.stream and options.subject):
+        parser.error('You must specify a stream/subject or at least one recipient.')
+
+    client = zulip.init_from_options(options)
+
+    if not options.message:
+        options.message = sys.stdin.read()
+
+    if options.stream:
+        message_data = {
+            'type': 'stream',
+            'content': options.message,
+            'subject': options.subject,
+            'to': options.stream,
+        }
+    else:
+        message_data = {
+            'type': 'private',
+            'content': options.message,
+            'to': recipients,
+        }
+
+    if not do_send_message(client, message_data):
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

api/examples/create-user

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+from os import path
+import optparse
+
+usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
+
+Create a user. You must be a realm admin to use this API, and the user
+will be created in your realm.
+
+Example: create-user --site=http://localhost:9991 --user=rwbarton@example.com --new-email=jarthur@example.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
+"""
+
+sys.path.append(path.join(path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--new-email')
+parser.add_option('--new-password')
+parser.add_option('--new-full-name')
+parser.add_option('--new-short-name')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.create_user({
+    'email': options.new_email,
+    'password': options.new_password,
+    'full_name': options.new_full_name,
+    'short_name': options.new_short_name
+}))

api/examples/edit-message

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
+
+Edits a message that you sent
+
+Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--message-id', default="")
+parser.add_option('--subject', default="")
+parser.add_option('--content', default="")
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "message_id": options.message_id,
+}
+if options.subject != "":
+    message_data["subject"] = options.subject
+if options.content != "":
+    message_data["content"] = options.content
+print(client.update_message(message_data))

api/examples/get-public-streams

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out all the public streams in the realm.
+
+Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_streams(include_public=True, include_subscribed=False))

api/examples/list-members

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
+
+List the names and e-mail addresses of the people in your realm.
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+for user in client.get_members()["members"]:
+    print(user["full_name"], user["email"])

api/examples/list-subscriptions

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out a list of the user's subscriptions.
+
+Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.list_subscriptions())

api/examples/print-events

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out certain events received by the indicated bot or user matching the filter below.
+
+Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_event(event):
+    # type: (Dict[str, Any]) -> None
+    print(event)
+
+# This is a blocking call, and will continuously poll for new events
+# Note also the filter here is messages to the stream Denmark; if you
+# don't specify event_types it'll print all events.
+client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])

api/examples/print-messages

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out each message received by the indicated bot or user.
+
+Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_message(message):
+    # type: (Dict[str, Any]) -> None
+    print(message)
+
+# This is a blocking call, and will continuously poll for new messages
+client.call_on_each_message(print_message)

api/examples/recent-messages

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages received by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+request = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.call_endpoint(
+    url='messages',
+    method='GET',
+    request=request,
+)
+
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])

api/examples/send-message

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
+
+Sends a test message to the specified recipients.
+
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--subject', default="test")
+parser.add_option('--message', default="test message")
+parser.add_option('--type', default='private')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if len(args) == 0:
+    parser.error("You must specify recipients")
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "type": options.type,
+    "content": options.message,
+    "subject": options.subject,
+    "to": args,
+}
+print(client.send_message(message_data))

api/examples/subscribe

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is subscribed to the listed streams.
+
+Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))

api/examples/unsubscribe

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is not subscribed to the listed streams.
+
+Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.remove_subscriptions(options.streams.split()))

api/examples/upload-file

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import importlib
+import optparse
+import sys
+
+from six.moves import StringIO as _StringIO
+sys.path.insert(0, './api')
+from typing import IO
+import zulip
+
+class StringIO(_StringIO):
+    name = '' # https://github.com/python/typeshed/issues/598
+
+usage = """upload-file --user=<user's email address> --api-key=<user's api key> [options]
+
+Upload a file, and print the corresponding URI.
+
+Example: upload-file --user=cordelia@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --file-path=cat.png
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+If no --file-path is specified, a placeholder text file will be used instead.
+"""
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--file-path')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+file = None # type: IO
+if options.file_path:
+    file = open(options.file_path, 'rb')
+else:
+    file = StringIO('This is a test file.')
+    file.name = 'test.txt'
+
+response = client.upload_file(file)
+
+try:
+    print('File URI: {}'.format(response['uri']))
+except KeyError:
+    print('Error! API response was: {}'.format(response))

api/examples/zuliprc

@@ -0,0 +1,4 @@
+; Save this file as ~/.zuliprc
+[api]
+key=<your bot's api key from the web interface>
+email=<your bot's email address>

api/integrations/asana/zulip_asana_config.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+### REQUIRED CONFIGURATION ###
+
+# Change these values to your Asana credentials.
+ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# Change these values to the credentials for your Asana bot.
+ZULIP_USER = "asana-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The Zulip stream that will receive Asana task updates.
+ZULIP_STREAM_NAME = "asana"
+
+
+### OPTIONAL CONFIGURATION ###
+
+# Set to None for logging to stdout when testing, and to a file for
+# logging in production.
+#LOG_FILE = "/var/tmp/zulip_asana.log"
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_asana.state"
+
+# When initially started, how many hours of messages to include.
+ASANA_INITIAL_HISTORY_HOURS = 1
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None

api/integrations/asana/zulip_asana_mirror

@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Asana integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_asana_mirror" script is run continuously, possibly on a work computer
+# or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+import base64
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional, Any, Tuple
+
+import json
+import logging
+import os
+import time
+from six.moves import urllib
+from six.moves.urllib import request as urllib_request
+import sys
+
+
+try:
+    import dateutil.parser
+    from dateutil.tz import gettz
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_asana_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+if config.LOG_FILE:
+    logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+else:
+    logging.basicConfig(level=logging.INFO)
+
+client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
+                      site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
+
+def fetch_from_asana(path):
+    # type: (str) -> Optional[Dict[str, Any]]
+    """
+    Request a resource through the Asana API, authenticating using
+    HTTP basic auth.
+    """
+    auth = base64.encodestring(b'%s:' % (config.ASANA_API_KEY,))
+    headers = {"Authorization": "Basic %s" % auth}
+
+    url = "https://app.asana.com/api/1.0" + path
+    request = urllib_request.Request(url, None, headers) # type: ignore
+    result = urllib_request.urlopen(request) # type: ignore
+
+    return json.load(result)
+
+def send_zulip(topic, content):
+    # type: (str, str) -> Dict[str, str]
+    """
+    Send a message to Zulip using the configured stream and bot credentials.
+    """
+    message = {"type": "stream",
+               "sender": config.ZULIP_USER,
+               "to": config.ZULIP_STREAM_NAME,
+               "subject": topic,
+               "content": content,
+               }
+    return client.send_message(message)
+
+def datestring_to_datetime(datestring):
+    # type: (str) -> datetime
+    """
+    Given an ISO 8601 datestring, return the corresponding datetime object.
+    """
+    return dateutil.parser.parse(datestring).replace(
+        tzinfo=gettz('Z'))
+
+class TaskDict(dict):
+    """
+    A helper class to turn a dictionary with task information into an
+    object where each of the keys is an attribute for easy access.
+    """
+    def __getattr__(self, field):
+        # type: (TaskDict, str) -> Any
+        return self.get(field)
+
+def format_topic(task, projects):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string that will be the Zulip message topic for this task.
+    """
+    # Tasks can be associated with multiple projects, but in practice they seem
+    # to mostly be associated with one.
+    project_name = projects[task.projects[0]["id"]]
+    return "%s: %s" % (project_name, task.name)
+
+def format_assignee(task, users):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string describing the task's assignee.
+    """
+    if task.assignee:
+        assignee_name = users[task.assignee["id"]]
+        assignee_info = "**Assigned to**: %s (%s)" % (
+            assignee_name, task.assignee_status)
+    else:
+        assignee_info = "**Status**: Unassigned"
+
+    return assignee_info
+
+def format_due_date(task):
+    # type: (TaskDict) -> str
+    """
+    Return a string describing the task's due date.
+    """
+    if task.due_on:
+        due_date_info = "**Due on**: %s" % (task.due_on,)
+    else:
+        due_date_info = "**Due date**: None"
+    return due_date_info
+
+def format_task_creation_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a newly-created task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** created:
+
+~~~ quote
+%s
+~~~
+
+%s
+%s
+""" % (task.name, task.notes, assignee_info, due_date_info)
+    return topic, content
+
+def format_task_completion_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a completed task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** completed. :white_check_mark:
+
+%s
+%s
+""" % (task.name, assignee_info, due_date_info)
+    return topic, content
+
+def since():
+    # type: () -> datetime
+    """
+    Return a newness threshold for task events to be processed.
+    """
+    # If we have a record of the last event processed and it is recent, use it,
+    # else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
+    def default_since():
+        # type: () -> datetime
+        return datetime.utcnow() - timedelta(
+            hours=config.ASANA_INITIAL_HISTORY_HOURS)
+
+    if os.path.exists(config.RESUME_FILE):
+        try:
+            with open(config.RESUME_FILE, "r") as f:
+                datestring = f.readline().strip()
+                timestamp = float(datestring)
+                max_timestamp_processed = datetime.fromtimestamp(timestamp)
+                logging.info("Reading from resume file: " + datestring)
+        except (ValueError, IOError) as e:
+            logging.warn("Could not open resume file: " + str(e))
+            max_timestamp_processed = default_since()
+    else:
+        logging.info("No resume file, processing an initial history.")
+        max_timestamp_processed = default_since()
+
+    # Even if we can read a timestamp from RESUME_FILE, if it is old don't use
+    # it.
+    return max(max_timestamp_processed, default_since())
+
+def process_new_events():
+    # type: () -> None
+    """
+    Forward new Asana task events to Zulip.
+    """
+    # In task queries, Asana only exposes IDs for projects and users, so we need
+    # to look up the mappings.
+    projects = dict((elt["id"], elt["name"]) for elt in
+                    fetch_from_asana("/projects")["data"])
+    users = dict((elt["id"], elt["name"]) for elt in
+                 fetch_from_asana("/users")["data"])
+
+    cutoff = since()
+    max_timestamp_processed = cutoff
+    time_operations = (("created_at", format_task_creation_event),
+                       ("completed_at", format_task_completion_event))
+    task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
+                   "modified_at", "due_on", "name", "notes", "projects"]
+
+    # First, gather all of the tasks that need processing. We'll
+    # process them in order.
+    new_events = []
+
+    for project_id in projects:
+        project_url = "/projects/%d/tasks?opt_fields=%s" % (
+            project_id, ",".join(task_fields))
+        tasks = fetch_from_asana(project_url)["data"]
+
+        for task in tasks:
+            task = TaskDict(task)
+
+            for time_field, operation in time_operations:
+                if task[time_field]:
+                    operation_time = datestring_to_datetime(task[time_field])
+                    if operation_time > cutoff:
+                        new_events.append((operation_time, time_field, operation, task))
+
+    new_events.sort()
+    now = datetime.utcnow()
+
+    for operation_time, time_field, operation, task in new_events:
+        # Unfortunately, creating an Asana task is not an atomic operation. If
+        # the task was just created, or is missing basic information, it is
+        # probably because the task is still being filled out -- wait until the
+        # next round to process it.
+        if (time_field == "created_at") and \
+                (now - operation_time < timedelta(seconds=30)):
+            # The task was just created, give the user some time to fill out
+            # more information.
+            return
+
+        if (time_field == "created_at") and (not task.name) and \
+                (now - operation_time < timedelta(seconds=60)):
+            # If this new task hasn't had a name for a full 30 seconds, assume
+            # you don't plan on giving it one.
+            return
+
+        topic, content = operation(task, projects, users)
+        logging.info("Sending Zulip for " + topic)
+        result = send_zulip(topic, content)
+
+        # If the Zulip wasn't sent successfully, don't update the
+        # max timestamp processed so the task has another change to
+        # be forwarded. Exit, giving temporary issues time to
+        # resolve.
+        if not result.get("result"):
+            logging.warn("Malformed result, exiting:")
+            logging.warn(str(result))
+            return
+
+        if result["result"] != "success":
+            logging.warn(result["msg"])
+            return
+
+        if operation_time > max_timestamp_processed:
+            max_timestamp_processed = operation_time
+
+    if max_timestamp_processed > cutoff:
+        max_datestring = max_timestamp_processed.strftime("%s.%f")
+        logging.info("Updating resume file: " + max_datestring)
+        open(config.RESUME_FILE, 'w').write(max_datestring)
+
+while True:
+    try:
+        process_new_events()
+        time.sleep(5)
+    except KeyboardInterrupt:
+        logging.info("Shutting down...")
+        logging.info("Set LOG_FILE to log to a file instead of stdout.")
+        break

api/integrations/basecamp/zulip_basecamp_config.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for basecamp account
+BASECAMP_ACCOUNT_ID = "12345678"
+BASECAMP_USERNAME = "foo@example.com"
+BASECAMP_PASSWORD = "p455w0rd"
+
+# This script will mirror this many hours of history on the first run.
+# On subsequent runs this value is ignored.
+BASECAMP_INITIAL_HISTORY_HOURS = 0
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "basecamp-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+ZULIP_STREAM_NAME = "basecamp"
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_basecamp.state"

api/integrations/basecamp/zulip_basecamp_mirror

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Basecamp activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "basecamp-mirror.py" script is run continuously, possibly on a work computer
+# or preferably on a server.
+# You may need to install the python-requests library.
+
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import re
+import sys
+from stderror import write
+import os
+from datetime import datetime, timedelta
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_basecamp_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+from six.moves.html_parser import HTMLParser
+from typing import Any, Dict
+import six
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipBasecamp/" + VERSION)
+user_agent = "Basecamp To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+htmlParser = HTMLParser()
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # type: () -> None
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr.write("Could not open up log for writing:")
+            sys.stderr.write(str(e))
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE),)
+        sys.stderr.write(str(e))
+
+# builds the message dict for sending a message with the Zulip API
+def build_message(event):
+    # type: (Dict[str, Any]) -> Dict[str, Any]
+    if not ('bucket' in event and 'creator' in event and 'html_url' in event):
+        logging.error("Perhaps the Basecamp API changed behavior? "
+                      "This event doesn't have the expected format:\n%s" % (event,))
+        return None
+    # adjust the topic length to be bounded to 60 characters
+    topic = event['bucket']['name']
+    if len(topic) > 60:
+        topic = topic[0:57] + "..."
+    # get the action and target values
+    action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
+    target = htmlParser.unescape(event.get('target', ''))
+    # Some events have "excerpts", which we blockquote
+    excerpt = htmlParser.unescape(event.get('excerpt', ''))
+    if excerpt.strip() == "":
+        message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
+    else:
+        message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
+    # assemble the message data dict
+    message_data = {
+        "type": "stream",
+        "to": config.ZULIP_STREAM_NAME,
+        "subject": topic,
+        "content": message,
+    }
+    return message_data
+
+# the main run loop for this mirror script
+def run_mirror():
+    # type: () -> None
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    try:
+        with open(config.RESUME_FILE) as f:
+            since = f.read() # type: Any
+        since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
+        assert since, "resume file does not meet expected format"
+        since = since.string
+    except (AssertionError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (e,))
+        since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
+    try:
+        # we use an exponential backoff approach when we get 429 (Too Many Requests).
+        sleepInterval = 1
+        while True:
+            time.sleep(sleepInterval)
+            response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
+                                    params={'since': since},
+                                    auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
+                                    headers = {"User-Agent": user_agent})
+            if response.status_code == 200:
+                sleepInterval = 1
+                events = json.loads(response.text)
+                if len(events):
+                    logging.info("Got event(s): %s" % (response.text,))
+            if response.status_code >= 500:
+                logging.error(str(response.status_code))
+                continue
+            if response.status_code == 429:
+                # exponential backoff
+                sleepInterval *= 2
+                logging.error(str(response.status_code))
+                continue
+            if response.status_code == 400:
+                logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
+                sys.exit(-1)
+            if response.status_code == 401:
+                logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
+                sys.exit(-1)
+            if len(events):
+                since = events[0]['created_at']
+            for event in reversed(events):
+                message_data = build_message(event)
+                if not message_data:
+                    continue
+                zulip_api_result = client.send_message(message_data)
+                if zulip_api_result['result'] == "success":
+                    logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
+                else:
+                    logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
+                # update 'since' each time in case we get KeyboardInterrupted
+                since = event['created_at']
+                # avoid hitting rate-limit
+                time.sleep(0.2)
+
+    except KeyboardInterrupt:
+        logging.info("Shutting down, please hold")
+        open("events.last", 'w').write(since)
+        logging.info("Done!")
+
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr.write("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    run_mirror()

api/integrations/codebase/zulip_codebase_config.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for your codebase account
+# Note that this is the Codebase API Username, found in the Settings page
+# for your account
+CODEBASE_API_USERNAME = "foo@example.com"
+CODEBASE_API_KEY = "1234561234567abcdef"
+
+# The URL of your codebase setup
+CODEBASE_ROOT_URL = "https://YOUR_COMPANY.codebasehq.com"
+
+# When initially started, how many hours of messages to include.
+# Note that the Codebase API only returns the 20 latest events,
+# if you have more than 20 events that fit within this window,
+# earlier ones may be lost
+CODEBASE_INITIAL_HISTORY_HOURS = 12
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "codebase-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The streams to send commit information and ticket information to
+ZULIP_COMMITS_STREAM_NAME = "codebase"
+ZULIP_TICKETS_STREAM_NAME = "tickets"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_codebase.state"

api/integrations/codebase/zulip_codebase_mirror

@@ -0,0 +1,332 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Codebase HQ activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_codebase_mirror" script is run continuously, possibly on a work
+# computer or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import sys
+import os
+
+from datetime import datetime, timedelta
+
+try:
+    import dateutil.parser
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_codebase_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import six
+import zulip
+from typing import Any, List, Dict, Optional
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipCodebase/" + VERSION)
+user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+def make_api_call(path):
+    # type: (str) -> Optional[List[Dict[str, Any]]]
+    response = requests.get("https://api3.codebasehq.com/%s" % (path,),
+                            auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
+                            params={'raw': True},
+                            headers = {"User-Agent": user_agent,
+                                       "Content-Type": "application/json",
+                                       "Accept": "application/json"})
+    if response.status_code == 200:
+        return json.loads(response.text)
+
+    if response.status_code >= 500:
+        logging.error(str(response.status_code))
+        return None
+    if response.status_code == 403:
+        logging.error("Bad authorization from Codebase. Please check your credentials")
+        sys.exit(-1)
+    else:
+        logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
+        return None
+
+def make_url(path):
+    # type: (str) -> str
+    return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
+
+def handle_event(event):
+    # type: (Dict[str, Any]) -> None
+    event = event['event']
+    event_type = event['type']
+    actor_name = event['actor_name']
+
+    raw_props = event.get('raw_properties', {})
+
+    project_link = raw_props.get('project_permalink')
+
+    subject = None
+    content = None
+    if event_type == 'repository_creation':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        project_name = raw_props.get('name')
+        project_repo_type = raw_props.get('scm_type')
+
+        url = make_url("projects/%s" % (project_link,))
+        scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
+
+        subject = "Repository %s Created" % (project_name,)
+        content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
+    elif event_type == 'push':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        num_commits = raw_props.get('commits_count')
+        branch = raw_props.get('ref_name')
+        project = raw_props.get('project_name')
+        repo_link = raw_props.get('repository_permalink')
+        deleted_ref = raw_props.get('deleted_ref')
+        new_ref = raw_props.get('new_ref')
+
+        subject = "Push to %s on %s" % (branch, project)
+
+        if deleted_ref:
+            content = "%s deleted branch %s from %s" % (actor_name, branch, project)
+        else:
+            if new_ref:
+                branch = "new branch %s" % (branch,)
+            content = ("%s pushed %s commit(s) to %s in project %s:\n\n" %
+                       (actor_name, num_commits, branch, project))
+            for commit in raw_props.get('commits'):
+                ref = commit.get('ref')
+                url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
+                message = commit.get('message')
+                content += "* [%s](%s): %s\n" % (ref, url, message)
+    elif event_type == 'ticketing_ticket':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        assignee = raw_props.get('assignee')
+        priority = raw_props.get('priority')
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+
+        if assignee is None:
+            assignee = "no one"
+        subject = "#%s: %s" % (num, name)
+        content = ("""%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" %
+                   (actor_name, num, url, priority, assignee, name))
+    elif event_type == 'ticketing_note':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        body = raw_props.get('content')
+        changes = raw_props.get('changes')
+
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+        subject = "#%s: %s" % (num, name)
+
+        content = ""
+        if body is not None and len(body) > 0:
+            content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
+
+        if 'status_id' in changes:
+            status_change = changes.get('status_id')
+            content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
+    elif event_type == 'ticketing_milestone':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        name = raw_props.get('name')
+        identifier = raw_props.get('identifier')
+        url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
+
+        subject = name
+        content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
+    elif event_type == 'comment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        comment = raw_props.get('content')
+        commit = raw_props.get('commit_ref')
+
+        # If there's a commit id, it's a comment to a commit
+        if commit:
+            repo_link = raw_props.get('repository_permalink')
+
+            url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
+
+            subject = "%s commented on %s" % (actor_name, commit)
+            content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
+        else:
+            # Otherwise, this is a Discussion item, and handle it
+            subj = raw_props.get("subject")
+            category = raw_props.get("category")
+            comment_content = raw_props.get("content")
+
+            subject = "Discussion: %s" % (subj,)
+
+            if category:
+                format_str = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~"
+                content = format_str % (actor_name, category, comment_content)
+            else:
+                content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
+
+    elif event_type == 'deployment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        start_ref = raw_props.get('start_ref')
+        end_ref = raw_props.get('end_ref')
+        environment = raw_props.get('environment')
+        servers = raw_props.get('servers')
+        repo_link = raw_props.get('repository_permalink')
+
+        start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
+        end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
+        between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (
+            project_link, repo_link, start_ref, end_ref))
+
+        subject = "Deployment to %s" % (environment,)
+
+        content = ("%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." %
+                   (actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment))
+        if servers is not None:
+            content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
+
+    elif event_type == 'named_tree':
+        # Docs say named_tree type used for new/deleting branches and tags,
+        # but experimental testing showed that they were all sent as 'push' events
+        pass
+    elif event_type == 'wiki_page':
+        logging.warn("Wiki page notifications not yet implemented")
+    elif event_type == 'sprint_creation':
+        logging.warn("Sprint notifications not yet implemented")
+    elif event_type == 'sprint_ended':
+        logging.warn("Sprint notifications not yet implemented")
+    else:
+        logging.info("Unknown event type %s, ignoring!" % (event_type,))
+
+    if subject and content:
+        if len(subject) > 60:
+            subject = subject[:57].rstrip() + '...'
+
+        res = client.send_message({"type": "stream",
+                                   "to": stream,
+                                   "subject": subject,
+                                   "content": content})
+        if res['result'] == 'success':
+            logging.info("Successfully sent Zulip with id: %s" % (res['id']))
+        else:
+            logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
+
+
+# the main run loop for this mirror script
+def run_mirror():
+    # type: () -> None
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    def default_since():
+        # type: () -> datetime
+        return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
+
+    try:
+        with open(config.RESUME_FILE) as f:
+            timestamp = f.read()
+        if timestamp == '':
+            since = default_since()
+        else:
+            since = datetime.fromtimestamp(float(timestamp))
+    except (ValueError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (str(e)))
+        since = default_since()
+
+    try:
+        sleepInterval = 1
+        while True:
+            events = make_api_call("activity")[::-1]
+            if events is not None:
+                sleepInterval = 1
+                for event in events:
+                    timestamp = event.get('event', {}).get('timestamp', '')
+                    event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None)
+                    if event_date > since:
+                        handle_event(event)
+                        since = event_date
+            else:
+                # back off a bit
+                if sleepInterval < 22:
+                    sleepInterval += 4
+            time.sleep(sleepInterval)
+
+    except KeyboardInterrupt:
+        open(config.RESUME_FILE, 'w').write(since.strftime("%s"))
+        logging.info("Shutting down Codebase mirror")
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # type: () -> None
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr.write("Could not open up log for writing:")
+            sys.stderr.write(str(e))
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr.write("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
+        sys.stderr.write(str(e))
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr.write("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+    run_mirror()

api/integrations/git/post-receive

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-receive hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-receive" script is run after receive-pack has accepted a pack
+# and the repository has been updated.  It is passed arguments in through
+# stdin in the form
+#  <oldrev> <newrev> <refname>
+# For example:
+#  aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
+
+from __future__ import absolute_import
+from typing import Text
+import os
+import sys
+import subprocess
+import os.path
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_git_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipGit/" + VERSION)
+
+def git_repository_name():
+    # type: () -> Text
+    output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
+    if output.strip() == "true":
+        return os.path.basename(os.getcwd())[:-len(".git")]
+    else:
+        return os.path.basename(os.path.dirname(os.getcwd()))
+
+def git_commit_range(oldrev, newrev):
+    # type: (str, str) -> str
+    log_cmd = ["git", "log", "--reverse",
+               "--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
+    commits = ''
+    for ln in subprocess.check_output(log_cmd).splitlines():
+        author_email, commit_id, subject  = ln.split(None, 2)
+        if hasattr(config, "format_commit_message"):
+            commits += config.format_commit_message(author_email, subject, commit_id)
+        else:
+            commits += '!avatar(%s) %s\n' % (author_email, subject)
+    return commits
+
+def send_bot_message(oldrev, newrev, refname):
+    # type: (str, str, str) -> None
+    repo_name   = git_repository_name()
+    branch      = refname.replace('refs/heads/', '')
+    destination = config.commit_notice_destination(repo_name, branch, newrev)
+    if destination is None:
+        # Don't forward the notice anywhere
+        return
+
+    new_head = newrev[:12]
+    old_head = oldrev[:12]
+
+    if (oldrev == '0000000000000000000000000000000000000000' or
+            newrev == '0000000000000000000000000000000000000000'):
+        # New branch pushed or old branch removed
+        added = ''
+        removed = ''
+    else:
+        added   = git_commit_range(oldrev, newrev)
+        removed = git_commit_range(newrev, oldrev)
+
+    if oldrev == '0000000000000000000000000000000000000000':
+        message = '`%s` was pushed to new branch `%s`' % (new_head, branch)
+    elif newrev == '0000000000000000000000000000000000000000':
+        message = 'branch `%s` was removed (was `%s`)' % (branch, old_head)
+    elif removed:
+        message = '`%s` was pushed to `%s`, **REMOVING**:\n\n%s' % (new_head, branch, removed)
+        if added:
+            message += '\n**and adding**:\n\n' + added
+        message += '\n**A HISTORY REWRITE HAS OCCURRED!**'
+        message += '\n@everyone: Please check your local branches to deal with this.'
+    elif added:
+        message = '`%s` was deployed to `%s` with:\n\n%s' % (new_head, branch, added)
+    else:
+        message = '`%s` was pushed to `%s`... but nothing changed?' % (new_head, branch)
+
+    message_data = {
+        "type": "stream",
+        "to": destination["stream"],
+        "subject": destination["subject"],
+        "content": message,
+    }
+    client.send_message(message_data)
+
+for ln in sys.stdin:
+    oldrev, newrev, refname = ln.strip().split()
+    send_bot_message(oldrev, newrev, refname)

api/integrations/git/zulip_git_config.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "git-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * repo   = the name of the git repository
+# * branch = the name of the branch that was pushed to
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit pushed to "master" to
+# * stream "commits"
+# * topic "master"
+# And similarly for branch "test-post-receive" (for use when testing).
+def commit_notice_destination(repo, branch, commit):
+    if branch in ["master", "test-post-receive"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (branch,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+# Modify this function to change how commits are displayed; the most
+# common customization is to include a link to the commit in your
+# graphical repository viewer, e.g.
+#
+# return '!avatar(%s) [%s](https://example.com/commits/%s)\n' % (author, subject, commit_id)
+def format_commit_message(author, subject, commit_id):
+    return '!avatar(%s) %s\n' % (author, subject)
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://zulip.example.com"

api/integrations/google/get-google-credentials

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import datetime
+import httplib2
+import os
+
+from oauth2client import client
+from oauth2client import tools
+from oauth2client.file import Storage
+
+try:
+    import argparse
+    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
+except ImportError:
+    flags = None
+
+# If modifying these scopes, delete your previously saved credentials
+# at zulip/bots/gcal/
+# NOTE: When adding more scopes, add them after the previous one in the same field, with a space
+# seperating them.
+SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
+# This file contains the information that google uses to figure out which application is requesting
+# this client's data.
+CLIENT_SECRET_FILE = 'client_secret.json'
+APPLICATION_NAME = 'Zulip Calendar Bot'
+HOME_DIR = os.path.expanduser('~')
+
+def get_credentials():
+    # type: () -> client.Credentials
+    """Gets valid user credentials from storage.
+
+    If nothing has been stored, or if the stored credentials are invalid,
+    the OAuth2 flow is completed to obtain the new credentials.
+
+    Returns:
+        Credentials, the obtained credential.
+    """
+
+    credential_path = os.path.join(HOME_DIR,
+                                   'google-credentials.json')
+
+    store = Storage(credential_path)
+    credentials = store.get()
+    if not credentials or credentials.invalid:
+        flow = client.flow_from_clientsecrets(os.path.join(HOME_DIR, CLIENT_SECRET_FILE), SCOPES)
+        flow.user_agent = APPLICATION_NAME
+        if flags:
+            # This attempts to open an authorization page in the default web browser, and asks the user
+            # to grant the bot access to their data. If the user grants permission, the run_flow()
+            # function returns new credentials.
+            credentials = tools.run_flow(flow, store, flags)
+        else:  # Needed only for compatibility with Python 2.6
+            credentials = tools.run(flow, store)
+        print('Storing credentials to ' + credential_path)
+
+get_credentials()

api/integrations/google/google-calendar

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import datetime
+import httplib2
+import itertools
+import logging
+import optparse
+import os
+from six.moves import urllib
+import sys
+import time
+import traceback
+from typing import List, Set, Tuple, Iterable, Optional
+
+from oauth2client import client, tools
+from oauth2client.file import Storage
+try:
+    from googleapiclient import discovery
+except ImportError:
+    logging.exception('Install google-api-python-client')
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../'))
+import zulip
+
+SCOPES = 'https://www.googleapis.com/auth/calendar.readonly'
+CLIENT_SECRET_FILE = 'client_secret.json'
+APPLICATION_NAME = 'Zulip'
+HOME_DIR = os.path.expanduser('~')
+
+# Our cached view of the calendar, updated periodically.
+events = []  # type: List[Tuple[int, datetime.datetime, str]]
+
+# Unique keys for events we've already sent, so we don't remind twice.
+sent = set()  # type: Set[Tuple[int, datetime.datetime]]
+
+sys.path.append(os.path.dirname(__file__))
+
+parser = optparse.OptionParser(r"""
+
+%prog \
+    --user foo@zulip.com \
+    --calendar calendarID@example.calendar.google.com
+
+    This integration can be used to send yourself reminders, on Zulip, of Google Calendar Events.
+
+    Before running this integration make sure you run the get-google-credentials file to give Zulip
+    access to certain aspects of your Google Account.
+
+    This integration should be run on your local machine. Your API key and other information are
+    revealed to local users through the command line.
+
+    Depends on: google-api-python-client
+""")
+
+
+parser.add_option('--interval',
+                  dest='interval',
+                  default=30,
+                  type=int,
+                  action='store',
+                  help='Minutes before event for reminder [default: 30]',
+                  metavar='MINUTES')
+
+parser.add_option('--calendar',
+                  dest = 'calendarID',
+                  default = 'primary',
+                  type = str,
+                  action = 'store',
+                  help = 'Calendar ID for the calendar you want to receive reminders from.')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+
+(options, args) = parser.parse_args()
+
+if not (options.zulip_email):
+    parser.error('You must specify --user')
+
+zulip_client = zulip.init_from_options(options)
+
+def get_credentials():
+    # type: () -> client.Credentials
+    """Gets valid user credentials from storage.
+
+    If nothing has been stored, or if the stored credentials are invalid,
+    an exception is thrown and the user is informed to run the script in this directory to get
+    credentials.
+
+    Returns:
+        Credentials, the obtained credential.
+    """
+    try:
+        credential_path = os.path.join(HOME_DIR,
+                                       'google-credentials.json')
+
+        store = Storage(credential_path)
+        credentials = store.get()
+
+        return credentials
+    except client.Error:
+        logging.exception('Error while trying to open the `google-credentials.json` file.')
+    except IOError:
+        logging.error("Run the get-google-credentials script from this directory first.")
+
+
+def get_events():
+    # type: () -> Iterable[Tuple[int, datetime.datetime, str]]
+    credentials = get_credentials()
+    creds = credentials.authorize(httplib2.Http())
+    service = discovery.build('calendar', 'v3', http=creds)
+
+    now = datetime.datetime.utcnow().isoformat() + 'Z' # 'Z' indicates UTC time
+    feed = service.events().list(calendarId=options.calendarID, timeMin=now, maxResults=5,
+                                 singleEvents=True, orderBy='startTime').execute()
+
+    for event in feed["items"]:
+        try:
+            start = event["start"]["dateTime"]
+        except KeyError:
+            start = event["start"]["date"]
+        start = start[:19]
+        # All-day events can have only a date
+        fmt = '%Y-%m-%dT%H:%M:%S' if 'T' in start else '%Y-%m-%d'
+        start = datetime.datetime.strptime(start, fmt)
+        try:
+            yield (event["id"], start, event["summary"])
+        except KeyError:
+            yield (event["id"], start, "(No Title)")
+
+
+def send_reminders():
+    # type: () -> Optional[None]
+    global sent
+
+    messages = []
+    keys = set()
+    now = datetime.datetime.now()
+
+    for id, start, summary in events:
+        dt = start - now
+        if dt.days == 0 and dt.seconds < 60 * options.interval:
+            # The unique key includes the start time, because of
+            # repeating events.
+            key = (id, start)
+            if key not in sent:
+                if start.hour == 0 and start.minute == 0:
+                    line = '%s is today.' % (summary,)
+                else:
+                    line = '%s starts at %s' % (summary, start.strftime('%H:%M'))
+                print('Sending reminder:', line)
+                messages.append(line)
+                keys.add(key)
+
+    if not messages:
+        return
+
+    if len(messages) == 1:
+        message = 'Reminder: ' + messages[0]
+    else:
+        message = 'Reminder:\n\n' + '\n'.join('* ' + m for m in messages)
+
+    zulip_client.send_message(dict(
+        type    = 'private',
+        to      = options.zulip_email,
+        sender  = options.zulip_email,
+        content = message))
+
+    sent.update(keys)
+
+# Loop forever
+for i in itertools.count():
+    try:
+        # We check reminders every minute, but only
+        # download the calendar every 10 minutes.
+        if not i % 10:
+            events = list(get_events())
+        send_reminders()
+    except:
+        logging.exception("Couldn't download Google calendar and/or couldn't post to Zulip.")
+    time.sleep(60)

api/integrations/hg/zulip-changegroup.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip hook for Mercurial changeset pushes.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+#
+# This hook is called when changesets are pushed to the master repository (ie
+# `hg push`). See https://zulipchat.com/integrations for installation instructions.
+from __future__ import absolute_import
+
+import zulip
+from six.moves import range
+from typing import Any, Optional, Text
+from mercurial import ui, repo
+
+VERSION = "0.9"
+
+def format_summary_line(web_url, user, base, tip, branch, node):
+    # type: (str, str, int, int, str, Text) -> Text
+    """
+    Format the first line of the message, which contains summary
+    information about the changeset and links to the changelog if a
+    web URL has been configured:
+
+    Jane Doe <jane@example.com> pushed 1 commit to master (170:e494a5be3393):
+    """
+    revcount = tip - base
+    plural = "s" if revcount > 1 else ""
+
+    if web_url:
+        shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
+        summary_url = "{shortlog}{tip}?revcount={revcount}".format(
+            shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
+        formatted_commit_count = "[{revcount} commit{s}]({url})".format(
+            revcount=revcount, s=plural, url=summary_url)
+    else:
+        formatted_commit_count = "{revcount} commit{s}".format(
+            revcount=revcount, s=plural)
+
+    return u"**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
+        user=user, commits=formatted_commit_count, branch=branch, tip=tip,
+        node=node[:12])
+
+def format_commit_lines(web_url, repo, base, tip):
+    # type: (str, repo, int, int) -> str
+    """
+    Format the per-commit information for the message, including the one-line
+    commit summary and a link to the diff if a web URL has been configured:
+    """
+    if web_url:
+        rev_base_url = web_url.rstrip("/") + "/rev/"
+
+    commit_summaries = []
+    for rev in range(base, tip):
+        rev_node = repo.changelog.node(rev)
+        rev_ctx = repo.changectx(rev_node)
+        one_liner = rev_ctx.description().split("\n")[0]
+
+        if web_url:
+            summary_url = rev_base_url + str(rev_ctx)
+            summary = "* [{summary}]({url})".format(
+                summary=one_liner, url=summary_url)
+        else:
+            summary = "* {summary}".format(summary=one_liner)
+
+        commit_summaries.append(summary)
+
+    return "\n".join(summary for summary in commit_summaries)
+
+def send_zulip(email, api_key, site, stream, subject, content):
+    # type: (str, str, str, str, str, Text) -> str
+    """
+    Send a message to Zulip using the provided credentials, which should be for
+    a bot in most cases.
+    """
+    client = zulip.Client(email=email, api_key=api_key,
+                          site=site,
+                          client="ZulipMercurial/" + VERSION)
+
+    message_data = {
+        "type": "stream",
+        "to": stream,
+        "subject": subject,
+        "content": content,
+    }
+
+    client.send_message(message_data)
+
+def get_config(ui, item):
+    # type: (ui, str) -> Optional[str]
+    try:
+        # configlist returns everything in lists.
+        return ui.configlist('zulip', item)[0]
+    except IndexError:
+        return None
+
+def hook(ui, repo, **kwargs):
+    # type: (ui, repo, Optional[Text]) -> None
+    """
+    Invoked by configuring a [hook] entry in .hg/hgrc.
+    """
+    hooktype = kwargs["hooktype"]
+    node = kwargs["node"]
+
+    ui.debug("Zulip: received {hooktype} event\n".format(hooktype=hooktype))
+
+    if hooktype != "changegroup":
+        ui.warn("Zulip: {hooktype} not supported\n".format(hooktype=hooktype))
+        exit(1)
+
+    ctx = repo.changectx(node)
+    branch = ctx.branch()
+
+    # If `branches` isn't specified, notify on all branches.
+    branch_whitelist = get_config(ui, "branches")
+    branch_blacklist = get_config(ui, "ignore_branches")
+
+    if branch_whitelist:
+        # Only send notifications on branches we are watching.
+        watched_branches = [b.lower().strip() for b in branch_whitelist.split(",")]
+        if branch.lower() not in watched_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    if branch_blacklist:
+        # Don't send notifications for branches we've ignored.
+        ignored_branches = [b.lower().strip() for b in branch_blacklist.split(",")]
+        if branch.lower() in ignored_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    # The first and final commits in the changeset.
+    base = repo[node].rev()
+    tip = len(repo)
+
+    email = get_config(ui, "email")
+    api_key = get_config(ui, "api_key")
+    site = get_config(ui, "site")
+
+    if not (email and api_key):
+        ui.warn("Zulip: missing email or api_key configurations\n")
+        ui.warn("in the [zulip] section of your .hg/hgrc.\n")
+        exit(1)
+
+    stream = get_config(ui, "stream")
+    # Give a default stream if one isn't provided.
+    if not stream:
+        stream = "commits"
+
+    web_url = get_config(ui, "web_url")
+    user = ctx.user()
+    content = format_summary_line(web_url, user, base, tip, branch, node)
+    content += format_commit_lines(web_url, repo, base, tip)
+
+    subject = branch
+
+    ui.debug("Sending to Zulip:\n")
+    ui.debug(content + "\n")
+
+    send_zulip(email, api_key, site, stream, subject, content)

api/integrations/jira/org/humbug/jira/ZulipListener.groovy

@@ -0,0 +1,149 @@
+/*
+* Copyright (c) 2014 Zulip, Inc
+*/
+
+package org.zulip.jira
+
+import static com.atlassian.jira.event.type.EventType.*
+
+import com.atlassian.jira.event.issue.AbstractIssueEventListener
+import com.atlassian.jira.event.issue.IssueEvent
+
+import java.util.logging.Level
+import java.util.logging.Logger
+
+import org.apache.commons.httpclient.HttpClient
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.methods.PostMethod
+import org.apache.commons.httpclient.NameValuePair
+
+class ZulipListener extends AbstractIssueEventListener {
+    Logger LOGGER = Logger.getLogger(ZulipListener.class.getName());
+
+    // The email address of one of the bots you created on your Zulip settings page.
+    String zulipEmail = ""
+    // That bot's API key.
+    String zulipAPIKey = ""
+
+    // What stream to send messages to. Must already exist.
+    String zulipStream = "jira"
+
+    // The base JIRA url for browsing
+    String issueBaseUrl = "https://jira.COMPANY.com/browse/"
+
+    // Your zulip domain
+    String base_url = "https://zulip.example.com/"
+
+    @Override
+    void workflowEvent(IssueEvent event) {
+      processIssueEvent(event)
+    }
+
+    String processIssueEvent(IssueEvent event) {
+      String author = event.user.displayName
+      String issueId = event.issue.key
+      String issueUrl = issueBaseUrl + issueId
+      String issueUrlMd = String.format("[%s](%s)", issueId, issueBaseUrl + issueId)
+      String title = event.issue.summary
+      String subject = truncate(String.format("%s: %s", issueId, title), 60)
+      String assignee = "no one"
+      if (event.issue.assignee) {
+        assignee = event.issue.assignee.name
+      }
+      String comment = "";
+      if (event.comment) {
+        comment = event.comment.body
+      }
+
+      String content;
+
+      // Event types:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/event/type/EventType.html
+      // Issue API:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/issue/Issue.html
+      switch (event.getEventTypeId()) {
+        case ISSUE_COMMENTED_ID:
+          content = String.format("%s **updated** %s with comment:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        case ISSUE_CREATED_ID:
+          content = String.format("%s **created** %s priority %s, assigned to @**%s**: \n\n> %s",
+                                  author, issueUrlMd, event.issue.priorityObject.name,
+                                  assignee, title)
+          break
+        case ISSUE_ASSIGNED_ID:
+          content = String.format("%s **reassigned** %s to **%s**",
+                                  author, issueUrlMd, assignee)
+          break
+        case ISSUE_DELETED_ID:
+          content = String.format("%s **deleted** %s!",
+                                  author, issueUrlMd)
+          break
+        case ISSUE_RESOLVED_ID:
+          content = String.format("%s **resolved** %s as %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_CLOSED_ID:
+          content = String.format("%s **closed** %s with resolution %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_REOPENED_ID:
+          content = String.format("%s **reopened** %s:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        default:
+          return
+      }
+
+      sendStreamMessage(zulipStream, subject, content)
+    }
+
+    String post(String method, NameValuePair[] parameters) {
+      PostMethod post = new PostMethod(zulipUrl(method))
+      post.setRequestHeader("Content-Type", post.FORM_URL_ENCODED_CONTENT_TYPE)
+      // TODO: Include more useful data in the User-agent
+      post.setRequestHeader("User-agent", "ZulipJira/0.1")
+      try {
+        post.setRequestBody(parameters)
+        HttpClient client = new HttpClient()
+        client.executeMethod(post)
+        String response = post.getResponseBodyAsString()
+        if (post.getStatusCode() != HttpStatus.SC_OK) {
+          String params = ""
+          for (NameValuePair pair: parameters) {
+              params += "\n" + pair.getName() + ":" + pair.getValue()
+          }
+          LOGGER.log(Level.SEVERE, "Error sending Zulip message:\n" + response + "\n\n" +
+                                   "We sent:" + params)
+        }
+        return response;
+      } catch (IOException e) {
+        throw new RuntimeException(e)
+      } finally {
+        post.releaseConnection()
+      }
+    }
+
+    String truncate(String string, int length) {
+      if (string.length() < length) {
+        return string
+      }
+      return string.substring(0, length - 3) + "..."
+    }
+
+    String sendStreamMessage(String stream, String subject, String message) {
+      NameValuePair[] body = [new NameValuePair("api-key", zulipAPIKey),
+                              new NameValuePair("email",   zulipEmail),
+                              new NameValuePair("type",    "stream"),
+                              new NameValuePair("to",      stream),
+                              new NameValuePair("subject", subject),
+                              new NameValuePair("content", message)]
+      return post("send_message", body);
+    }
+
+    String zulipUrl(method) {
+      return base_url.replaceAll("/+$", "") + "/api/v1/" + method
+    }
+}

api/integrations/nagios/nagios-notify-zulip

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+import optparse
+import zulip
+
+from typing import List, Text, Dict, Any
+
+VERSION = "0.9"
+# Nagios passes the notification details as command line options.
+# In Nagios, "output" means "first line of output", and "long
+# output" means "other lines of output".
+parser = optparse.OptionParser()  # type: optparse.OptionParser
+parser.add_option('--output', default='')
+parser.add_option('--long-output', default='')
+parser.add_option('--stream', default='nagios')
+parser.add_option('--config', default='/etc/nagios3/zuliprc')
+for opt in ('type', 'host', 'service', 'state'):
+    parser.add_option('--' + opt)
+(opts, args) = parser.parse_args()  # type: Any, List[Text]
+
+client = zulip.Client(config_file=opts.config,
+                      client="ZulipNagios/" + VERSION)  # type: zulip.Client
+
+msg = dict(type='stream', to=opts.stream)  # type: Dict[str, Any]
+
+# Set a subject based on the host or service in question.  This enables
+# threaded discussion of multiple concurrent issues, and provides useful
+# context when narrowed.
+#
+# We send PROBLEM and RECOVERY messages to the same subject.
+if opts.service is None:
+    # Host notification
+    thing = 'host'  # type: Text
+    msg['subject'] = 'host %s' % (opts.host,)
+else:
+    # Service notification
+    thing = 'service'
+    msg['subject'] = 'service %s on %s' % (opts.service, opts.host)
+
+if len(msg['subject']) > 60:
+    msg['subject'] = msg['subject'][0:57].rstrip() + "..."
+# e.g. **PROBLEM**: service is CRITICAL
+msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
+
+# The "long output" can contain newlines represented by "\n" escape sequences.
+# The Nagios mail command uses /usr/bin/printf "%b" to expand these.
+# We will be more conservative and handle just this one escape sequence.
+output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip()  # type: Text
+if output:
+    # Put any command output in a code block.
+    msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
+
+client.send_message(msg)

api/integrations/nagios/zulip_nagios.cfg

@@ -0,0 +1,21 @@
+define contact{
+        contact_name                    zulip
+        alias                           zulip
+        service_notification_period     24x7
+        host_notification_period        24x7
+        service_notification_options    w,u,c,r
+        host_notification_options       d,r
+        service_notification_commands   notify-service-by-zulip
+        host_notification_commands      notify-host-by-zulip
+        }
+
+# Zulip commands
+define command {
+    command_name    notify-host-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
+}
+
+define command {
+    command_name    notify-service-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
+}

api/integrations/nagios/zuliprc.example

@@ -0,0 +1,5 @@
+# Fill these values in with the appropriate values for your realm, and
+# then install this value at /etc/nagios3/zuliprc
+[api]
+email = nagios-bot@example.com
+key = 0123456789abcdef0123456789abcdef

api/integrations/openshift/post_deploy

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-receive hook.
+# Copyright © 2012-2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import os
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_openshift_config as config
+VERSION = '0.1'
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client='ZulipOpenShift/' + VERSION)
+
+def get_deployment_details():
+    # type: () -> Dict[str, str]
+    # "gear deployments" output example:
+    #   Activation time - Deployment ID - Git Ref - Git SHA1
+    #   2017-01-07 15:40:30 -0500 - 9e2b7143 - master - b9ce57c - ACTIVE
+    dep = subprocess.check_output(['gear', 'deployments']).splitlines()[1]
+    splits = dep.split(' - ')
+
+    return dict(app_name=os.environ['OPENSHIFT_APP_NAME'],
+                url=os.environ['OPENSHIFT_APP_DNS'],
+                branch=splits[2],
+                commit_id=splits[3])
+
+def send_bot_message(deployment):
+    # type: (Dict[str, str]) -> None
+    destination = config.deployment_notice_destination(deployment['branch'])
+    if destination is None:
+        # No message should be sent
+        return
+    message = config.format_deployment_message(**deployment)
+
+    client.send_message({
+        'type': 'stream',
+        'to': destination['stream'],
+        'subject': destination['subject'],
+        'content': message,
+    })
+
+    return
+
+deployment = get_deployment_details()
+send_bot_message(deployment)

api/integrations/openshift/zulip_openshift_config.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# https://github.com/python/mypy/issues/1141
+from typing import Text
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = 'openshift-bot@example.com'
+ZULIP_API_KEY = '0123456789abcdef0123456789abcdef'
+
+# deployment_notice_destination() lets you customize where deployment notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * branch = the name of the branch where the deployed commit was
+#            pushed to
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit pushed to "master" to
+# * stream "deployments"
+# * topic "master"
+# And similarly for branch "test-post-receive" (for use when testing).
+def deployment_notice_destination(branch):
+    # type: (str) -> Dict[str, Text]
+    if branch in ['master', 'test-post-receive']:
+        return dict(stream  = 'deployments',
+                    subject = u'%s' % (branch,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+# Modify this function to change how deployments are displayed
+#
+# It takes the following arguments:
+# * app_name  = the name of the app being deployed
+# * url       = the FQDN (Fully Qualified Domain Name) where the app
+#                can be found
+# * branch    = the name of the branch where the deployed commit was
+#                pushed to
+# * commit_id = hash of the commit that triggered the deployment
+# * dep_id    = deployment id
+# * dep_time  = deployment timestamp
+def format_deployment_message(
+        app_name='', url='', branch='', commit_id='', dep_id='', dep_time=''):
+    # type: (str, str, str, str, str, str) -> str
+    return 'Deployed commit `%s` (%s) in [%s](%s)' % (
+        commit_id, branch, app_name, url)
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None # type: str
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = 'https://zulip.example.com'

api/integrations/perforce/license.txt

@@ -0,0 +1,26 @@
+git_p4.py was downloaded from https://raw.github.com/git/git/34022ba/git-p4.py
+
+The header of that file references <http://opensource.org/licenses/mit-license.php>,
+the textual contents of which are included below.
+
+The MIT License (MIT)
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

api/integrations/perforce/zulip_change-commit.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+'''Zulip notification change-commit hook.
+
+In Perforce, The "change-commit" trigger is fired after a metadata has been
+created, files have been transferred, and the changelist committed to the depot
+database.
+
+This specific trigger expects command-line arguments in the form:
+  %change% %changeroot%
+
+For example:
+  1234 //depot/security/src/
+
+'''
+from __future__ import print_function
+
+import os
+import sys
+import os.path
+
+import git_p4
+
+__version__ = "0.1"
+
+sys.path.insert(0, os.path.dirname(__file__))
+from typing import Any, Dict, Optional, Text
+import zulip_perforce_config as config
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipPerforce/" + __version__)  # type: zulip.Client
+
+try:
+    changelist = int(sys.argv[1])  # type: int
+    changeroot = sys.argv[2]  # type: str
+except IndexError:
+    print("Wrong number of arguments.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+except ValueError:
+    print("First argument must be an integer.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+
+metadata = git_p4.p4_describe(changelist)  # type: Dict[str, str]
+
+destination = config.commit_notice_destination(changeroot, changelist)  # type: Optional[Dict[str, str]]
+if destination is None:
+    # Don't forward the notice anywhere
+    sys.exit(0)
+
+message = "**{0}** committed revision @{1} to `{2}`.\n\n> {3}".format(
+    metadata["user"],
+    metadata["change"],
+    changeroot,
+    metadata["desc"])  # type: str
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}  # type: Dict[str, Any]
+client.send_message(message_data)

api/integrations/perforce/zulip_perforce_config.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "p4-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+ZULIP_SITE = "https://zulip.example.com"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the Perforce depot on the server
+# * changelist = the changelist id
+#
+# Returns a dictionary encoding the stream and topic to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for ones in the
+# "master-plan" and "secret" subdirectories of //depot/ to:
+# * stream "depot_subdirectory-commits"
+# * subject "change_root"
+def commit_notice_destination(path, changelist):
+    dirs = path.split('/')
+    if len(dirs) >= 4 and dirs[3] not in ("*", "..."):
+        directory = dirs[3]
+    else:
+        # No subdirectory, so just use "depot"
+        directory = dirs[2]
+
+    if directory not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "%s-commits" % (directory,),
+                    subject = path)
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None

api/integrations/rss/rss-bot

@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# RSS integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import calendar
+import errno
+import hashlib
+from six.moves.html_parser import HTMLParser
+import logging
+import optparse
+import os
+import sys
+import time
+from six.moves import urllib
+from typing import Dict, List, Tuple, Any
+
+import feedparser
+import zulip
+VERSION = "0.9" # type: str
+RSS_DATA_DIR = os.path.expanduser(os.path.join('~', '.cache', 'zulip-rss')) # type: str
+OLDNESS_THRESHOLD = 30 # type: int
+
+usage = """Usage: Send summaries of RSS entries for your favorite feeds to Zulip.
+
+This bot requires the feedparser module.
+
+To use this script:
+
+1. Create an RSS feed file containing 1 feed URL per line (default feed
+   file location: ~/.cache/zulip-rss/rss-feeds)
+2. Subscribe to the stream that will receive RSS updates (default stream: rss)
+3. create a ~/.zuliprc as described on https://zulipchat.com/api#api_keys
+4. Test the script by running it manually, like this:
+
+/usr/local/share/zulip/integrations/rss/rss-bot
+
+You can customize the location on the feed file and recipient stream, e.g.:
+
+/usr/local/share/zulip/integrations/rss/rss-bot --feed-file=/path/to/my-feeds --stream=my-rss-stream
+
+4. Configure a crontab entry for this script. A sample crontab entry for
+processing feeds stored in the default location and sending to the default
+stream every 5 minutes is:
+
+*/5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
+
+parser = optparse.OptionParser(usage) # type: optparse.OptionParser
+parser.add_option('--stream',
+                  dest='stream',
+                  help='The stream to which to send RSS messages.',
+                  default="rss",
+                  action='store')
+parser.add_option('--data-dir',
+                  dest='data_dir',
+                  help='The directory where feed metadata is stored',
+                  default=os.path.join(RSS_DATA_DIR),
+                  action='store')
+parser.add_option('--feed-file',
+                  dest='feed_file',
+                  help='The file containing a list of RSS feed URLs to follow, one URL per line',
+                  default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
+                  action='store')
+parser.add_option_group(zulip.generate_option_group(parser))
+(opts, args) = parser.parse_args() # type: Tuple[Any, List[str]]
+
+def mkdir_p(path):
+    # type: (str) -> None
+    # Python doesn't have an analog to `mkdir -p` < Python 3.2.
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+try:
+    mkdir_p(opts.data_dir)
+except OSError:
+    # We can't write to the logfile, so just print and give up.
+    print("Unable to store RSS data at %s." % (opts.data_dir,), file=sys.stderr)
+    exit(1)
+
+log_file = os.path.join(opts.data_dir, "rss-bot.log") # type: str
+log_format = "%(asctime)s: %(message)s" # type: str
+logging.basicConfig(format=log_format)
+
+formatter = logging.Formatter(log_format) # type: logging.Formatter
+file_handler = logging.FileHandler(log_file) # type: logging.FileHandler
+file_handler.setFormatter(formatter)
+
+logger = logging.getLogger(__name__) # type: logging.Logger
+logger.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+def log_error_and_exit(error):
+    # type: (str) -> None
+    logger.error(error)
+    logger.error(usage)
+    exit(1)
+
+class MLStripper(HTMLParser):
+    def __init__(self):
+        # type: () -> None
+        self.reset()
+        self.fed = [] # type: List[str]
+
+    def handle_data(self, data):
+        # type: (str) -> None
+        self.fed.append(data)
+
+    def get_data(self):
+        # type: () -> str
+        return ''.join(self.fed)
+
+def strip_tags(html):
+    # type: (str) -> str
+    stripper = MLStripper()
+    stripper.feed(html)
+    return stripper.get_data()
+
+def compute_entry_hash(entry):
+    # type: (Dict[str, Any]) -> str
+    entry_time = entry.get("published", entry.get("updated"))
+    entry_id = entry.get("id", entry.get("link"))
+    return hashlib.md5(entry_id + str(entry_time)).hexdigest()
+
+def elide_subject(subject):
+    # type: (str) -> str
+    MAX_TOPIC_LENGTH = 60
+    if len(subject) > MAX_TOPIC_LENGTH:
+        subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
+    return subject
+
+def send_zulip(entry, feed_name):
+    # type: (Any, str) -> Dict[str, Any]
+    content = "**[%s](%s)**\n%s\n%s" % (entry.title,
+                                        entry.link,
+                                        strip_tags(entry.summary),
+                                        entry.link) # type: str
+    message = {"type": "stream",
+               "sender": opts.zulip_email,
+               "to": opts.stream,
+               "subject": elide_subject(feed_name),
+               "content": content,
+               } # type: Dict[str, str]
+    return client.send_message(message)
+
+try:
+    with open(opts.feed_file, "r") as f:
+        feed_urls = [feed.strip() for feed in f.readlines()] # type: List[str]
+except IOError:
+    log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
+
+client = zulip.Client(email=opts.zulip_email, api_key=opts.zulip_api_key,
+                      site=opts.zulip_site, client="ZulipRSS/" + VERSION) # type: zulip.Client
+
+first_message = True # type: bool
+
+for feed_url in feed_urls:
+    feed_file = os.path.join(opts.data_dir, urllib.parse.urlparse(feed_url).netloc) # Type: str
+
+    try:
+        with open(feed_file, "r") as f:
+            old_feed_hashes = dict((line.strip(), True) for line in f.readlines()) # type: Dict[str, bool]
+    except IOError:
+        old_feed_hashes = {}
+
+    new_hashes = [] # type: List[str]
+    data = feedparser.parse(feed_url) # type: feedparser.parse
+
+    for entry in data.entries:
+        entry_hash = compute_entry_hash(entry) # type: str
+        # An entry has either been published or updated.
+        entry_time = entry.get("published_parsed", entry.get("updated_parsed")) # type: Tuple[int, int]
+        if entry_time is not None and (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
+            # As a safeguard against misbehaving feeds, don't try to process
+            # entries older than some threshold.
+            continue
+        if entry_hash in old_feed_hashes:
+            # We've already seen this. No need to process any older entries.
+            break
+        if (not old_feed_hashes) and (len(new_hashes) >= 3):
+            # On a first run, pick up the 3 most recent entries. An RSS feed has
+            # entries in reverse chronological order.
+            break
+
+        feed_name = data.feed.title or feed_url # type: str
+
+        response = send_zulip(entry, feed_name) # type: Dict[str, Any]
+        if response["result"] != "success":
+            logger.error("Error processing %s" % (feed_url,))
+            logger.error(str(response))
+            if first_message:
+                # This is probably some fundamental problem like the stream not
+                # existing or something being misconfigured, so bail instead of
+                # getting the same error for every RSS entry.
+                log_error_and_exit("Failed to process first message")
+        # Go ahead and move on -- perhaps this entry is corrupt.
+        new_hashes.append(entry_hash)
+        first_message = False
+
+    with open(feed_file, "a") as f:
+        for hash in new_hashes:
+            f.write(hash + "\n")
+
+    logger.info("Sent zulips for %d %s entries" % (len(new_hashes), feed_url))

api/integrations/svn/post-commit

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-commit hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-commit" script is run after a transaction is completed and a new
+# revision is created.  It is passed arguments on the command line in this
+# form:
+#  <path> <revision>
+# For example:
+# /srv/svn/carols 1843
+
+import os
+import sys
+import os.path
+import pysvn
+
+if False:
+    from typing import Any, Dict, List, Optional, Text, Tuple, Union
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_svn_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipSVN/" + VERSION)  # type: zulip.Client
+svn = pysvn.Client()  # type: pysvn.Client
+
+path, rev = sys.argv[1:]  # type: Tuple[Text, Text]
+
+# since its a local path, prepend "file://"
+path = "file://" + path
+
+entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0]  # type: Dict[Text, Union[Text, pysvn.Revision, List[Dict[Text, pysvn.Revision]]]]
+message = "**{0}** committed revision r{1} to `{2}`.\n\n> {3}".format(
+    entry['author'],
+    rev,
+    path.split('/')[-1],
+    entry['revprops']['svn:log'])  # type: Text
+
+destination = config.commit_notice_destination(path, rev)  # type: Optional[Dict[Text, Text]]
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}  # type: Dict[str, Any]
+client.send_message(message_data)

api/integrations/svn/zulip_svn_config.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "svn-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the svn repository on the server
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for the "evil-master-plan"
+# and "my-super-secret-repository" repos to
+# * stream "commits"
+# * topic "branch_name"
+def commit_notice_destination(path, commit):
+    repo = path.split('/')[-1]
+    if repo not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (repo,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://zulip.example.com"

api/integrations/trac/zulip_trac.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Zulip trac plugin -- sends zulips when tickets change.
+#
+# Install by copying this file and zulip_trac_config.py to the trac
+# plugins/ subdirectory, customizing the constants in
+# zulip_trac_config.py, and then adding "zulip_trac" to the
+# components section of the conf/trac.ini file, like so:
+#
+# [components]
+# zulip_trac = enabled
+#
+# You may then need to restart trac (or restart Apache) for the bot
+# (or changes to the bot) to actually be loaded by trac.
+
+from trac.core import Component, implements
+from trac.ticket import ITicketChangeListener
+import sys
+import os.path
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_trac_config as config
+VERSION = "0.9"
+
+if False:
+    from typing import Any
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipTrac/" + VERSION)
+
+def markdown_ticket_url(ticket, heading="ticket"):
+    # type: (Any, str) -> str
+    return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
+
+def markdown_block(desc):
+    # type: (str) -> str
+    return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
+
+def truncate(string, length):
+    # type: (str, int) -> str
+    if len(string) <= length:
+        return string
+    return string[:length - 3] + "..."
+
+def trac_subject(ticket):
+    # type: (Any) -> str
+    return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
+
+def send_update(ticket, content):
+    # type: (Any, str) -> None
+    client.send_message({
+        "type": "stream",
+        "to": config.STREAM_FOR_NOTIFICATIONS,
+        "content": content,
+        "subject": trac_subject(ticket)
+    })
+
+class ZulipPlugin(Component):
+    implements(ITicketChangeListener)
+
+    def ticket_created(self, ticket):
+        # type: (Any) -> None
+        """Called when a ticket is created."""
+        content = "%s created %s in component **%s**, priority **%s**:\n" % \
+            (ticket.values.get("reporter"), markdown_ticket_url(ticket),
+             ticket.values.get("component"), ticket.values.get("priority"))
+        # Include the full subject if it will be truncated
+        if len(ticket.values.get("summary")) > 60:
+            content += "**%s**\n" % (ticket.values.get("summary"),)
+        if ticket.values.get("description") != "":
+            content += "%s" % (markdown_block(ticket.values.get("description")),)
+        send_update(ticket, content)
+
+    def ticket_changed(self, ticket, comment, author, old_values):
+        # type: (Any, str, str, Dict[str, Any]) -> None
+        """Called when a ticket is modified.
+
+        `old_values` is a dictionary containing the previous values of the
+        fields that have changed.
+        """
+        if not (set(old_values.keys()).intersection(set(config.TRAC_NOTIFY_FIELDS)) or
+                (comment and "comment" in set(config.TRAC_NOTIFY_FIELDS))):
+            return
+
+        content = "%s updated %s" % (author, markdown_ticket_url(ticket))
+        if comment:
+            content += ' with comment: %s\n\n' % (markdown_block(comment),)
+        else:
+            content += ":\n\n"
+        field_changes = []
+        for key in old_values.keys():
+            if key == "description":
+                content += '- Changed %s from %s\n\nto %s' % (key, markdown_block(old_values.get(key)),
+                                                              markdown_block(ticket.values.get(key)))
+            elif old_values.get(key) == "":
+                field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
+            elif ticket.values.get(key) == "":
+                field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
+            else:
+                field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
+                                                               ticket.values.get(key)))
+        content += ", ".join(field_changes)
+
+        send_update(ticket, content)
+
+    def ticket_deleted(self, ticket):
+        # type: (Any) -> None
+        """Called when a ticket is deleted."""
+        content = "%s was deleted." % markdown_ticket_url(ticket, heading="Ticket")
+        send_update(ticket, content)

api/integrations/trac/zulip_trac_config.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# See zulip_trac.py for installation and configuration instructions
+
+# Change these constants to configure the plugin:
+ZULIP_USER = "trac-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+STREAM_FOR_NOTIFICATIONS = "trac"
+TRAC_BASE_TICKET_URL = "https://trac.example.com/ticket"
+
+# Most people find that having every change in Trac result in a
+# notification is too noisy -- in particular, when someone goes
+# through recategorizing a bunch of tickets, that can often be noisy
+# and annoying.  We solve this issue by only sending a notification
+# for changes to the fields listed below.
+#
+# TRAC_NOTIFY_FIELDS lets you specify which fields will trigger a
+# Zulip notification in response to a trac update; you should change
+# this list to match your team's workflow.  The complete list of
+# possible fields is:
+#
+# (priority, milestone, cc, owner, keywords, component, severity,
+#  type, versions, description, resolution, summary, comment)
+TRAC_NOTIFY_FIELDS = ["description", "summary", "resolution", "comment", "owner"]
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"

api/integrations/twitter/twitter-bot

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Twitter integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import os
+import sys
+import optparse
+from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
+
+import zulip
+VERSION = "0.9"
+CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
+
+def write_config(config, since_id, user):
+    # type: (ConfigParser, int, int) -> None
+    config.set('twitter', 'since_id', since_id)
+    config.set('twitter', 'user_id', user)
+    with open(CONFIGFILE, 'wb') as configfile:
+        config.write(configfile)
+
+parser = optparse.OptionParser(r"""
+
+%prog --user foo@example.com --api-key 0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --twitter-id twitter_handle --site=https://zulip.example.com
+
+    Slurp tweets on your timeline into a specific zulip stream.
+
+    Run this on your personal machine.  Your API key and twitter id
+    are revealed to local users through the command line or config
+    file.
+
+    This bot uses OAuth to authenticate with twitter. Please create a
+    ~/.zulip_twitterrc with the following contents:
+
+    [twitter]
+    consumer_key =
+    consumer_secret =
+    access_token_key =
+    access_token_secret =
+
+    In order to obtain a consumer key & secret, you must register a
+    new application under your twitter account:
+
+    1. Go to http://dev.twitter.com
+    2. Log in
+    3. In the menu under your username, click My Applications
+    4. Create a new application
+
+    Make sure to go the application you created and click "create my
+    access token" as well.  Fill in the values displayed.
+
+    Depends on: https://github.com/bear/python-twitter version 3.1
+    (`pip install python-twitter`)
+""")
+
+parser.add_option('--twitter-id',
+                  help='Twitter username to poll for new tweets from"',
+                  metavar='URL')
+parser.add_option('--stream',
+                  help='Default zulip stream to write tweets to')
+parser.add_option('--limit-tweets',
+                  default=15,
+                  type='int',
+                  help='Maximum number of tweets to push at once')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if not options.twitter_id:
+    parser.error('You must specify --twitter-id')
+
+try:
+    config = ConfigParser()
+    config.read(CONFIGFILE)
+
+    consumer_key = config.get('twitter', 'consumer_key')
+    consumer_secret = config.get('twitter', 'consumer_secret')
+    access_token_key = config.get('twitter', 'access_token_key')
+    access_token_secret = config.get('twitter', 'access_token_secret')
+except (NoSectionError, NoOptionError):
+    parser.error("Please provide a ~/.zulip_twitterrc")
+
+if not consumer_key or not consumer_secret or not access_token_key or not access_token_secret:
+    parser.error("Please provide a ~/.zulip_twitterrc")
+
+try:
+    import twitter
+except ImportError:
+    parser.error("Please install twitter-python")
+
+api = twitter.Api(consumer_key=consumer_key,
+                  consumer_secret=consumer_secret,
+                  access_token_key=access_token_key,
+                  access_token_secret=access_token_secret)
+
+
+user = api.VerifyCredentials()
+
+if not user.id:
+    print("Unable to log in to twitter with supplied credentials. Please double-check and try again")
+    sys.exit()
+
+try:
+    since_id = config.getint('twitter', 'since_id')
+except NoOptionError:
+    since_id = -1
+
+try:
+    user_id = config.get('twitter', 'user_id')
+except NoOptionError:
+    user_id = options.twitter_id
+
+client = zulip.Client(
+    email=options.zulip_email,
+    api_key=options.zulip_api_key,
+    site=options.zulip_site,
+    client="ZulipTwitter/" + VERSION,
+    verbose=True)
+
+if since_id < 0 or options.twitter_id != user_id:
+    # No since id yet, fetch the latest and then start monitoring from next time
+    # Or, a different user id is being asked for, so start from scratch
+    # Either way, fetch last 5 tweets to start off
+    statuses = api.GetUserTimeline(screen_name=options.twitter_id, count=5)
+else:
+    # We have a saved last id, so insert all newer tweets into the zulip stream
+    statuses = api.GetUserTimeline(screen_name=options.twitter_id, since_id=since_id)
+
+for status in statuses[::-1][:options.limit_tweets]:
+    composed = "%s (%s)" % (status.user.name, status.user.screen_name)
+    message = {
+        "type": "stream",
+        "to": [options.stream],
+        "subject": composed,
+        "content": status.text,
+    }
+
+    ret = client.send_message(message)
+
+    if ret['result'] == 'error':
+        # If sending failed (e.g. no such stream), abort and retry next time
+        print("Error sending message to zulip: %s" % ret['msg'])
+        break
+    else:
+        since_id = status.id
+
+write_config(config, since_id, user_id)

api/integrations/twitter/twitter-search-bot

@@ -0,0 +1,194 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Twitter search integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import os
+import sys
+import optparse
+from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
+
+import zulip
+VERSION = "0.9"
+CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
+
+def write_config(config, since_id):
+    # type: (ConfigParser, int) -> None
+    if 'search' not in config.sections():
+        config.add_section('search')
+    config.set('search', 'since_id', since_id)
+    with open(CONFIGFILE, 'wb') as configfile:
+        config.write(configfile)
+
+parser = optparse.OptionParser(r"""
+
+%prog --user username@example.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+    --search="@nprnews,quantum physics"
+
+Send Twitter search results to a Zulip stream.
+
+Depends on: https://github.com/bear/python-twitter version 3.1
+
+To use this script:
+
+0. Use `pip install python-twitter` to install `python-twitter`.
+1. Set up Twitter authentication, as described below
+2. Subscribe to the stream that will receive Twitter updates (default stream: twitter)
+3. Test the script by running it manually, like this:
+
+/usr/local/share/zulip/integrations/twitter/twitter-search-bot \
+    --search="@nprnews,quantum physics" --site=https://zulip.example.com
+
+4. Configure a crontab entry for this script. A sample crontab entry
+that will process tweets every 5 minutes is:
+
+*/5 * * * * /usr/local/share/zulip/integrations/twitter/twitter-search-bot [options]
+
+== Setting up Twitter authentications ==
+
+Run this on a personal or trusted machine, because your API key is
+visible to local users through the command line or config file.
+
+This bot uses OAuth to authenticate with Twitter. Please create a
+~/.zulip_twitterrc with the following contents:
+
+[twitter]
+consumer_key =
+consumer_secret =
+access_token_key =
+access_token_secret =
+
+In order to obtain a consumer key & secret, you must register a
+new application under your Twitter account:
+
+1. Go to http://dev.twitter.com
+2. Log in
+3. In the menu under your username, click My Applications
+4. Create a new application
+
+Make sure to go the application you created and click "create my
+access token" as well. Fill in the values displayed.
+""")
+
+parser.add_option('--search',
+                  dest='search_terms',
+                  help='Terms to search on',
+                  action='store')
+parser.add_option('--stream',
+                  dest='stream',
+                  help='The stream to which to send tweets',
+                  default="twitter",
+                  action='store')
+parser.add_option('--limit-tweets',
+                  default=15,
+                  type='int',
+                  help='Maximum number of tweets to send at once')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+(opts, args) = parser.parse_args()
+
+if not opts.search_terms:
+    parser.error('You must specify a search term.')
+
+try:
+    config = ConfigParser()
+    config.read(CONFIGFILE)
+
+    consumer_key = config.get('twitter', 'consumer_key')
+    consumer_secret = config.get('twitter', 'consumer_secret')
+    access_token_key = config.get('twitter', 'access_token_key')
+    access_token_secret = config.get('twitter', 'access_token_secret')
+except (NoSectionError, NoOptionError):
+    parser.error("Please provide a ~/.zulip_twitterrc")
+
+if not (consumer_key and consumer_secret and access_token_key and access_token_secret):
+    parser.error("Please provide a ~/.zulip_twitterrc")
+
+try:
+    since_id = config.getint('search', 'since_id')
+except (NoOptionError, NoSectionError):
+    since_id = 0
+
+try:
+    import twitter
+except ImportError:
+    parser.error("Please install twitter-python")
+
+api = twitter.Api(consumer_key=consumer_key,
+                  consumer_secret=consumer_secret,
+                  access_token_key=access_token_key,
+                  access_token_secret=access_token_secret)
+
+user = api.VerifyCredentials()
+
+if not user.id:
+    print("Unable to log in to twitter with supplied credentials. Please double-check and try again")
+    sys.exit()
+
+client = zulip.Client(
+    email=opts.zulip_email,
+    api_key=opts.zulip_api_key,
+    site=opts.zulip_site,
+    client="ZulipTwitterSearch/" + VERSION,
+    verbose=True)
+
+search_query = " OR ".join(opts.search_terms.split(","))
+statuses = api.GetSearch(search_query, since_id=since_id)
+
+for status in statuses[::-1][:opts.limit_tweets]:
+    # https://twitter.com/eatevilpenguins/status/309995853408530432
+    composed = "%s (%s)" % (status.user.name,
+                            status.user.screen_name)
+    url = "https://twitter.com/%s/status/%s" % (status.user.screen_name,
+                                                status.id)
+    content = status.text
+
+    search_term_used = None
+    for term in opts.search_terms.split(","):
+        if term.lower() in content.lower():
+            search_term_used = term
+            break
+    # For some reason (perhaps encodings or message tranformations we
+    # didn't anticipate), we don't know what term was used, so use a
+    # default.
+    if not search_term_used:
+        search_term_used = "mentions"
+
+    message = {
+        "type": "stream",
+        "to": [opts.stream],
+        "subject": search_term_used,
+        "content": url,
+    }
+
+    ret = client.send_message(message)
+
+    if ret['result'] == 'error':
+        # If sending failed (e.g. no such stream), abort and retry next time
+        print("Error sending message to zulip: %s" % ret['msg'])
+        break
+    else:
+        since_id = status.id
+
+write_config(config, since_id)