Update changelog for Zulip 1.4.3 release.

streams: Fix autosubscribe security bug (CVE-2017-0881).
A bug in Zulip's implementation of the "stream exists" endpoint meant that any user of a Zulip server could subscribe to an invite-only stream without needing to be invited by using the "autosubscribe" argument. Thanks to Rafid Aslam for discovering this issue.
2025-10-27 01:53:59 +00:00 · 2017-01-29 15:09:02 -08:00 · 2017-01-29 15:09:02 -08:00 · 2016-09-27 20:09:20 -07:00 · 2016-09-27 20:09:20 -07:00 · 2016-09-27 20:01:17 -07:00
3326 changed files with 158703 additions and 289217 deletions
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,10 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        threshold: 0.03
-        base: auto
-    patch: off
--- a/.coveralls.yml
+++ b/.coveralls.yml
@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,19 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-charset = utf-8
-trim_trailing_whitespace = true
-insert_final_newline = true
-
-[*.{sh,py,js,json,yml,xml,css,md,markdown,handlebars,html}]
-indent_style = space
-indent_size = 4
-
-[*.{svg,rb,pp,pl}]
-indent_style = space
-indent_size = 2
-
-[*.{cfg}]
-indent_style = space
-indent_size = 8
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,2 +0,0 @@
-static/js/blueslip.js
-static/webpack-bundles
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,327 +0,0 @@
-{
-    "env": {
-        "node": true,
-        "es6": true
-    },
-    "parserOptions": {
-        "sourceType": "module"
-    },
-    "globals": {
-        "$": false,
-        "_": false,
-        "jQuery": false,
-        "Spinner": false,
-        "Handlebars": false,
-        "XDate": false,
-        "zxcvbn": false,
-        "LazyLoad": false,
-        "Dropbox": false,
-        "SockJS": false,
-        "marked": false,
-        "moment": false,
-        "i18n": false,
-        "DynamicText": false,
-        "LightboxCanvas": false,
-        "bridge": false,
-        "page_params": false,
-        "attachments_ui": false,
-        "csrf_token": false,
-        "typeahead_helper": false,
-        "pygments_data": false,
-        "popovers": false,
-        "server_events": false,
-        "server_events_dispatch": false,
-        "ui": false,
-        "ui_report": false,
-        "ui_util": false,
-        "lightbox": false,
-        "stream_color": false,
-        "people": false,
-        "navigate": false,
-        "settings_account": false,
-        "settings_display": false,
-        "settings_notifications": false,
-        "settings_muting": false,
-        "settings_lab": false,
-        "settings_bots": false,
-        "settings_sections": false,
-        "settings_emoji": false,
-        "settings_org": false,
-        "settings_users": false,
-        "settings_streams": false,
-        "settings_filters": false,
-        "settings": false,
-        "resize": false,
-        "loading": false,
-        "typing": false,
-        "typing_events": false,
-        "typing_data": false,
-        "typing_status": false,
-        "sent_messages": false,
-        "compose": false,
-        "compose_actions": false,
-        "compose_state": false,
-        "compose_fade": false,
-        "overlays": false,
-        "stream_create": false,
-        "stream_edit": false,
-        "subs": false,
-        "stream_muting": false,
-        "stream_events": false,
-        "timerender": false,
-        "message_live_update": false,
-        "message_edit": false,
-        "reload": false,
-        "composebox_typeahead": false,
-        "search": false,
-        "topic_list": false,
-        "topic_generator": false,
-        "gear_menu": false,
-        "hashchange": false,
-        "hash_util": false,
-        "message_list": false,
-        "Filter": false,
-        "pointer": false,
-        "util": false,
-        "MessageListView": false,
-        "blueslip": false,
-        "rows": false,
-        "WinChan": false,
-        "muting_ui": false,
-        "Socket": false,
-        "channel": false,
-        "components": false,
-        "message_viewport": false,
-        "upload_widget": false,
-        "avatar": false,
-        "realm_icon": false,
-        "feature_flags": false,
-        "search_suggestion": false,
-        "notifications": false,
-        "message_flags": false,
-        "bot_data": false,
-        "top_left_corner": false,
-        "stream_sort": false,
-        "stream_list": false,
-        "stream_popover": false,
-        "narrow_state": false,
-        "narrow": false,
-        "admin_sections": false,
-        "admin": false,
-        "stream_data": false,
-        "topic_data": false,
-        "list_util": false,
-        "muting": false,
-        "Dict": false,
-        "unread": false,
-        "alert_words_ui": false,
-        "message_store": false,
-        "message_util": false,
-        "message_events": false,
-        "message_fetch": false,
-        "favicon": false,
-        "condense": false,
-        "list_render": false,
-        "floating_recipient_bar": false,
-        "tab_bar": false,
-        "emoji": false,
-        "presence": false,
-        "activity": false,
-        "invite": false,
-        "colorspace": false,
-        "reactions": false,
-        "tutorial": false,
-        "templates": false,
-        "alert_words": false,
-        "fenced_code": false,
-        "markdown": false,
-        "echo": false,
-        "localstorage": false,
-        "localStorage": false,
-        "current_msg_list": true,
-        "home_msg_list": false,
-        "pm_list": false,
-        "pm_conversations": false,
-        "recent_senders": false,
-        "unread_ui": false,
-        "unread_ops": false,
-        "user_events": false,
-        "Plotly": false,
-        "emoji_codes": false,
-        "drafts": false,
-        "katex": false,
-        "Clipboard": false,
-        "emoji_picker": false,
-        "hotspots": false,
-        "compose_ui": false,
-        "common": false,
-        "desktop_notifications_panel": false
-    },
-    "rules": {
-        "array-callback-return": "error",
-        "array-bracket-spacing": "error",
-        "arrow-spacing": [ "error", { "before": true, "after": true } ],
-        "block-scoped-var": 2,
-        "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
-        "camelcase": 0,
-        "comma-dangle": [ "error",
-            {
-                "arrays": "always-multiline",
-                "objects": "always-multiline",
-                "imports": "always-multiline",
-                "exports": "always-multiline",
-                "functions": "never"
-            }
-        ],
-        "complexity": [ 0, 4 ],
-        "curly": 2,
-        "dot-notation": [ "error", { "allowKeywords": true } ],
-        "eol-last": [ "error", "always" ],
-        "eqeqeq": 2,
-        "func-style": [ "off", "expression" ],
-        "guard-for-in": 2,
-        "keyword-spacing": [ "error",
-            {
-                "before": true,
-                "after": true,
-                "overrides": {
-                    "return": { "after": true },
-                    "throw": { "after": true },
-                    "case": { "after": true }
-                }
-            }
-        ],
-        "max-depth": [ 0, 4 ],
-        "max-len": [ "error", 100, 2,
-            {
-                "ignoreUrls": true,
-                "ignoreComments": false,
-                "ignoreRegExpLiterals": true,
-                "ignoreStrings": true,
-                "ignoreTemplateLiterals": true
-            }
-        ],
-        "max-params": [ 0, 3 ],
-        "max-statements": [ 0, 10 ],
-        "new-cap": [ "error",
-            {
-                "newIsCap": true,
-                "capIsNew": false
-            }
-        ],
-        "new-parens": 2,
-        "newline-per-chained-call": 0,
-        "no-alert": 2,
-        "no-array-constructor": "error",
-        "no-bitwise": 2,
-        "no-caller": 2,
-        "no-case-declarations": "error",
-        "no-catch-shadow": 2,
-        "no-console": 0,
-        "no-const-assign": "error",
-        "no-control-regex": 2,
-        "no-debugger": 2,
-        "no-delete-var": 2,
-        "no-div-regex": 2,
-        "no-dupe-class-members": "error",
-        "no-dupe-keys": 2,
-        "no-duplicate-imports": "error",
-        "no-else-return": 2,
-        "no-empty": 2,
-        "no-empty-character-class": 2,
-        "no-eq-null": 2,
-        "no-eval": 2,
-        "no-ex-assign": 2,
-        "no-extra-parens": [ "error", "functions" ],
-        "no-extra-semi": 2,
-        "no-fallthrough": 2,
-        "no-floating-decimal": 2,
-        "no-func-assign": 2,
-        "no-implied-eval": 2,
-        "no-iterator": "error",
-        "no-label-var": 2,
-        "no-labels": 2,
-        "no-loop-func": 2,
-        "no-mixed-requires": [ 0, false ],
-        "no-multi-str": 2,
-        "no-native-reassign": 2,
-        "no-nested-ternary": 0,
-        "no-new-func": "error",
-        "no-new-object": 2,
-        "no-new-wrappers": 2,
-        "no-obj-calls": 2,
-        "no-octal": 2,
-        "no-octal-escape": 2,
-        "no-param-reassign": 0,
-        "no-plusplus": 2,
-        "no-proto": 2,
-        "no-redeclare": 2,
-        "no-regex-spaces": 2,
-        "no-restricted-syntax": 0,
-        "no-return-assign": 2,
-        "no-script-url": 2,
-        "no-self-compare": 2,
-        "no-shadow": 0,
-        "no-sync": 2,
-        "no-ternary": 0,
-        "no-undef": "error",
-        "no-undef-init": 2,
-        "no-underscore-dangle": 0,
-        "no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
-        "no-unreachable": 2,
-        "no-unused-expressions": 2,
-        "no-unused-vars": [ "error",
-            {
-                "vars": "local",
-                "args": "after-used",
-                "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
-            }
-        ],
-        "no-use-before-define": 2,
-        "no-useless-constructor": "error",
-        // The Zulip codebase complies partially with the "no-useless-escape"
-        // rule; only regex expressions haven't been updated yet.
-        // Updated regex expressions are currently being tested in casper
-        // files and will decide about a potential future enforcement of this rule.
-        "no-useless-escape": 0,
-        "no-whitespace-before-property": 0,
-        "no-with": 2,
-        "one-var": [ "error", "never" ],
-        "padded-blocks": 0,
-        "prefer-const": [ "error",
-            {
-                "destructuring": "any",
-                "ignoreReadBeforeAssign": true
-            }
-        ],
-        "quote-props": [ "error", "as-needed",
-            {
-                "keywords": false,
-                "unnecessary": true,
-                "numbers": false
-            }
-        ],
-        "quotes": [ 0, "single" ],
-        "radix": 2,
-        "semi": 2,
-        "space-before-blocks": 2,
-        "space-before-function-paren": [ "error",
-            {
-                "anonymous": "always",
-                "named": "never",
-                "asyncArrow": "always"
-            }
-        ],
-        "space-in-parens": 2,
-        "space-infix-ops": 0,
-        "spaced-comment": 0,
-        "strict": 0,
-        "template-curly-spacing": "error",
-        "unnecessary-strict": 0,
-        "use-isnan": 2,
-        "valid-typeof": [ "error", { "requireStringLiterals": true } ],
-        "wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
-        "wrap-regex": 0,
-        "yoda": 2
-    }
-}
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,11 +1,21 @@
-*   text=auto eol=lf
-*.gif binary
-*.jpg binary
-*.eot binary
-*.woff binary
-*.woff2 binary
-*.svg binary
-*.ttf binary
-*.png binary
-*.otf binary
-*.tif binary
+.gitignore export-ignore
+.gitattributes export-ignore
+/analytics export-ignore
+/assets export-ignore
+/bots export-ignore
+/corporate export-ignore
+/static export-ignore
+/tools export-ignore
+/zilencer export-ignore
+/templates/analytics export-ignore
+/templates/corporate export-ignore
+/templates/zilencer export-ignore
+/puppet/zulip_internal export-ignore
+/zproject/local_settings.py export-ignore
+/zproject/test_settings.py export-ignore
+/zerver/fixtures export-ignore
+/zerver/tests export-ignore
+/frontend_tests export-ignore
+/node_modules export-ignore
+/humbug export-ignore
+/locale export-ignore
--- a/.gitignore
+++ b/.gitignore
@@ -1,56 +1,26 @@
-# Quick format and style primer:
-#
-#  * If a pattern is meant only for a specific location, it should have a
-#    leading slash, like `/staticfiles.json`.
-#    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
-#      will do, but this makes a confusing pattern.  Adding a leading slash
-#      is clearer.
-#
-#  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
-#    match in any subdirectory.
-#
-#  * Subdirectories with several internal things to ignore get their own
-#    `.gitignore` files.
-#
-# See `git help ignore` for details on the format.
-
-## Config files for the dev environment
-/zproject/dev-secrets.conf
-/tools/conf.ini
-/tools/custom_provision
-
-## Byproducts of setting up and using the dev environment
 *.pyc
-
-/.vagrant
-/var
-
-# Static build
-*.mo
-npm-debug.log
-/node_modules
-/prod-static
-/staticfiles.json
-/webpack-stats-production.json
-/yarn-error.log
-
-# Test / analysis tools
-.coverage
-
-## Files left by various editors and local environments
-# (Ideally these should be in everyone's respective personal gitignore files.)
 *~
+/prod-static
+/errors/*
 *.sw[po]
-.idea
+*.DS_Store
+stats/
 .kdev4
+.idea
 zulip.kdev4
+coverage/
+.coverage
+/queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
-*.sublime-project
-*.sublime-workspace
-.vscode/
-*.DS_Store
-
-## Miscellaneous
-# (Ideally this section is empty.)
+.vagrant
+/zproject/dev-secrets.conf
+static/js/bundle.js
+static/third/gemoji/
+static/third/zxcvbn/
+static/locale/language_options.json
+node_modules
+npm-debug.log
+*.mo
+var/*
--- a/.gitlint
+++ b/.gitlint
@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing
-
-extra-path=tools/lib/gitlint-rules.py
-
-[title-match-regex]
-regex=^.+\.$
-
-[title-max-length]
-line-length=76
-
-[body-max-line-length]
-line-length=76
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,75 +1,47 @@
-# See https://zulip.readthedocs.io/en/latest/travis.html for
-# high-level documentation on our Travis CI setup.
 dist: trusty
+before_install:
+   - nvm install 0.10
 install:
-  # Disable broken riak sources.list in Travis base image 2017-10-18
-  - rm -vf "/etc/apt/sources.list.d/*riak*"
-
-  # Disable Travis CI's built-in NVM installation
-  - mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
-
-  # Install codecov, the library for the code coverage reporting tool we use
-  # With a retry to minimize impact of transient networking errors.
-  - mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
-
-  # This is the main setup job for the test suite
-  - mispipe "tools/travis/setup-$TEST_SUITE" ts
-
-  # Clean any caches that are not in use to avoid our cache
-  # becoming huge.
-  - mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
-
-script:
-  # We unset GEM_PATH here as a hack to work around Travis CI having
-  # broken running their system puppet with Ruby.  See
-  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
-  - unset GEM_PATH
-  - mispipe "./tools/travis/$TEST_SUITE" ts
+  - pip install coveralls
+  - tools/travis/setup-$TEST_SUITE
+  - tools/clean-venv-cache --travis
 cache:
-  yarn: true
-  apt: false
-  directories:
+  - apt: false
+  - directories:
+    - $HOME/phantomjs
    - $HOME/zulip-venv-cache
-    - $HOME/zulip-npm-cache
-    - $HOME/zulip-emoji-cache
+    - node_modules
    - $HOME/node
 env:
  global:
+    - COVERALLS_PARALLEL=true
+    - COVERALLS_SERVICE_NAME=travis-pro
+    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
    - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
 language: python
-# Our test suites generally run on Python 3.4, the version in
-# Ubuntu 14.04 trusty, which is the oldest OS release we support.
+python:
+  - "2.7"
+  - "3.4"
 matrix:
  include:
-    # Travis will actually run the jobs in the order they're listed here;
-    # that doesn't seem to be documented, but it's what we see empirically.
-    # We only get 4 jobs running at a time, so we try to make the first few
-    # the most likely to break.
-    - python: "3.4"
-      env: TEST_SUITE=frontend
-    - python: "3.4"
-      env: TEST_SUITE=backend
    - python: "3.4"
+      env: TEST_SUITE=static-analysis
+    - python: "2.7"
      env: TEST_SUITE=production
-    - python: "3.5"
-      env: TEST_SUITE=backend
+      sudo: required
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
 sudo: required
+services:
+- docker
 addons:
-  artifacts:
-    paths:
-      # Casper debugging data (screenshots, etc.) is super useful for
-      # debugging test flakes.
-      - $(ls var/casper/* | tr "\n" ":")
-      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
  postgresql: "9.3"
-  apt:
-    packages:
-      - moreutils
 after_success:
-  - codecov
+  coveralls
 notifications:
-  webhooks:
-    urls:
-      - https://zulip.org/zulipbot/travis
-    on_success: always
-    on_failure: always
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
--- a/.tx/config
+++ b/.tx/config
@@ -1,15 +1,17 @@
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant

 [zulip.djangopo]
 source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 file_filter = static/locale/<lang>/LC_MESSAGES/django.po
+lang_map = zh-Hans: zh_CN

 [zulip.translationsjson]
 source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
+lang_map = zh-Hans: zh-CN
+
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,105 +0,0 @@
-# Zulip Code of Conduct
-
-Like the technical community as a whole, the Zulip team and community is
-made up of a mixture of professionals and volunteers from all over the
-world, working on every aspect of the mission, including mentorship,
-teaching, and connecting people.
-
-Diversity is one of our huge strengths, but it can also lead to
-communication issues and unhappiness. To that end, we have a few ground
-rules that we ask people to adhere to. This code applies equally to
-founders, mentors, and those seeking help and guidance.
-
-This isn't an exhaustive list of things that you can't do. Rather, take it
-in the spirit in which it's intended --- a guide to make it easier to enrich
-all of us and the technical communities in which we participate.
-
-## Expected Behavior
-
-The following behaviors are expected and requested of all community members:
-
-* Participate. In doing so, you contribute to the health and longevity of
-  the community.
-* Exercise consideration and respect in your speech and actions.
-* Attempt collaboration before conflict. Assume good faith.
-* Refrain from demeaning, discriminatory, or harassing behavior and speech.
-* Take action or alert community leaders if you notice a dangerous
-  situation, someone in distress, or violations of this code, even if they
-  seem inconsequential.
-* Community event venues may be shared with members of the public; be
-  respectful to all patrons of these locations.
-
-## Unacceptable Behavior
-
-The following behaviors are considered harassment and are unacceptable
-within the Zulip community:
-
-* Jokes or derogatory language that singles out members of any race,
-  ethnicity, culture, national origin, color, immigration status, social and
-  economic class, educational level, language proficiency, sex, sexual
-  orientation, gender identity and expression, age, size, family status,
-  political belief, religion, and mental and physical ability.
-* Violence, threats of violence, or violent language directed against
-  another person.
-* Disseminating or threatening to disseminate another person's personal
-  information.
-* Personal insults of any sort.
-* Posting or displaying sexually explicit or violent material.
-* Inappropriate photography or recording.
-* Deliberate intimidation, stalking, or following (online or in person).
-* Unwelcome sexual attention. This includes sexualized comments or jokes,
-  inappropriate touching or groping, and unwelcomed sexual advances.
-* Sustained disruption of community events, including talks and
-  presentations.
-* Advocating for, or encouraging, any of the behaviors above.
-
-## Reporting and Enforcement
-
-Harassment and other code of conduct violations reduce the value of the
-community for everyone. If someone makes you or anyone else feel unsafe or
-unwelcome, please report it to the community organizers at
-zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
-report either personally or anonymously.
-
-If a community member engages in unacceptable behavior, the community
-organizers may take any action they deem appropriate, up to and including a
-temporary ban or permanent expulsion from the community without warning (and
-without refund in the case of a paid event).
-
-If someone outside the development community (e.g. a user of the Zulip
-software) engages in unacceptable behavior that affects someone in the
-community, we still want to know. Even if we don't have direct control over
-the violator, the community organizers can still support the people
-affected, reduce the chance of a similar violation in the future, and take
-any direct action we can.
-
-The nature of reporting means it can only help after the fact. If you see
-something you can do while a violation is happening, do it. A lot of the
-harms of harassment and other violations can be mitigated by the victim
-knowing that the other people present are on their side.
-
-All reports will be kept confidential. In some cases we may determine that a
-public statement will need to be made. In such cases, the identities of all
-victims and reporters will remain confidential unless those individuals
-instruct us otherwise.
-
-## Scope
-
-We expect all community participants (contributors, paid or otherwise,
-sponsors, and other guests) to abide by this Code of Conduct in all
-community venues, online and in-person, as well as in all private
-communications pertaining to community business.
-
-This Code of Conduct and its related procedures also applies to unacceptable
-behavior occurring outside the scope of community activities when such
-behavior has the potential to adversely affect the safety and well-being of
-community members.
-
-## License and Attribution
-
-This Code of Conduct is adapted from the
-[Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
-[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
-under a
-[Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
-license.
--- a/15
+++ b/15
@@ -0,0 +1,15 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+WORKDIR /srv/zulip
--- a/17
+++ b/17
@@ -1,17 +0,0 @@
-FROM ubuntu:trusty
-
-EXPOSE 9991
-
-RUN apt-get update && apt-get install -y wget
-
-RUN locale-gen en_US.UTF-8
-
-RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-USER zulip
-
-RUN ln -nsf /srv/zulip ~/zulip
-
-RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
-
-WORKDIR /srv/zulip
--- a/README.md
+++ b/README.md
@@ -17,77 +17,54 @@ previews, group private messages, audible notifications,
 missed-message emails, desktop apps, and much more.

 Further information on the Zulip project and its features can be found
-at <https://www.zulip.org>.
+at https://www.zulip.org.

-[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip)
-[![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip.svg)](https://codecov.io/gh/zulip/zulip)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)](http://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/)
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/)
-[![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](http://twitter.com/zulip)
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master)

 ## Community

 There are several places online where folks discuss Zulip.

-* The primary place is the
-  [Zulip development community Zulip server][czo-doc] at
-  chat.zulip.org.
+One of those places is our [public Zulip instance](https://zulip.tabbott.net/).
+You can go through the simple signup process at that link, and then you
+will soon be talking to core Zulip developers and other users.  To get
+help in real time, you will have the best luck finding core developers
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get answered
+within a day.

-* For Google Summer of Code students and applicants, we have
-[a mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc)
-for help, questions, and announcements.  But it's often simpler to
-[visit chat.zulip.org][czo-doc] instead.
+We have a [Google mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
+that is currently pretty low traffic.  It is where we do things like
+announce public meetings or major releases.  You can also use it to
+ask questions about features or possible bugs.

-* We have a [public development discussion mailing list][zulip-devel],
-zulip-devel, which is currently pretty low traffic because most
-discussions happen in our public Zulip instance.  We use it to
-announce Zulip developer community gatherings and ask for feedback on
-major technical or design decisions.  It has several hundred
-subscribers, so you can use it to ask questions about features or
-possible bugs, but please don't use it ask for generic help getting
-started as a contributor (e.g. because you want to do Google Summer of
-Code).  The rest of this page covers how to get involved in the Zulip
-project in detail.
-
-* Zulip also has a [blog](https://blog.zulip.org/) and
-  [twitter account](https://twitter.com/zulip).
-
-* Last but not least, we use [GitHub](https://github.com/zulip/zulip)
-to track Zulip-related issues (and store our code, of course).
-Anybody with a GitHub account should be able to create Issues there
-pertaining to bugs or enhancement requests.  We also use Pull Requests
-as our primary mechanism to receive code contributions.
-
-The Zulip community has a [Code of Conduct][code-of-conduct].
-
-[zulip-devel]: https://groups.google.com/forum/#!forum/zulip-devel
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
+track Zulip-related issues (and store our code, of course).
+Anybody with a Github account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull
+Requests as our primary mechanism to receive code contributions.

 ## Installing the Zulip Development environment

 The Zulip development environment is the recommended option for folks
-interested in trying out Zulip, since it is very easy to install.
-This is documented in [the developer installation guide][dev-install].
+interested in trying out Zulip.  This is documented in [the developer
+installation guide][dev-install].

 ## Running Zulip in production

-Zulip in production supports Ubuntu 16.04 Xenial and Ubuntu 14.04
-Trusty. We're happy to support work to enable Zulip to run on
-additional platforms. The installation process is
-[documented here](https://zulip.readthedocs.io/en/latest/prod.html).
+Zulip in production only supports Ubuntu 14.04 right now, but work is
+ongoing on adding support for additional platforms. The installation
+process is documented at https://zulip.org/server.html and in more
+detail in [the
+documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).

 ## Ways to contribute

-Zulip welcomes all forms of contributions!  This page documents the
+Zulip welcomes all forms of contributions!  The page documents the
 Zulip development process.

 * **Pull requests**. Before a pull request can be merged, you need to
-sign the [Dropbox Contributor License Agreement][cla].  Also,
+to sign the [Dropbox Contributor License Agreement][cla].  Also,
 please skim our [commit message style guidelines][doc-commit-style].
-We encourage early pull requests for work in progress. Prefix the title
-of your pull request with `[WIP]` and reference it when asking for
-community feedback. When you are ready for final review, remove
-the `[WIP]`.

 * **Testing**. The Zulip automated tests all run automatically when
 you submit a pull request, but you can also run them all in your
@@ -103,59 +80,48 @@ and [new feature tutorial][doc-newfeat]. You can also improve
 [Zulip.org][z-org].

 * **Mailing lists and bug tracker**. Zulip has a [development
-discussion mailing list](#community) and uses [GitHub issues
+discussion mailing list][gg-devel] and uses [GitHub issues
 ][gh-issues].  There are also lists for the [Android][email-android]
 and [iOS][email-ios] apps.  Feel free to send any questions or
 suggestions of areas where you'd love to see more documentation to the
-relevant list! Check out our [bug report guidelines][bug-report]
-before submitting. Please report any security issues you discover to
+relevant list!  Please report any security issues you discover to
 zulip-security@googlegroups.com.

 * **App codebases**. This repository is for the Zulip server and web
-app (including most integrations).  The
-[beta React Native mobile app][mobile], [Java Android app][Android]
-(see [our mobile strategy][mobile-strategy]),
-[new Electron desktop app][electron], and
-[legacy Qt-based desktop app][desktop] are all separate repositories.
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].

 * **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
 integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
-and [Trello][]), plus [node.js API bindings][node], an [isomorphic
- JavaScript library][zulip-js], and a [full-text search PostgreSQL
- extension][tsearch], as separate repos.
+and [Trello][]), plus [node.js API bindings][node], and a [full-text search
+PostgreSQL extension][tsearch], as separate repos.

 * **Translations**.  Zulip is in the process of being translated into
 10+ languages, and we love contributions to our translations.  See our
 [translating documentation][transifex] if you're interested in
 contributing!

-* **Code Reviews**. Zulip is all about community and helping each
-other out.  Check out [#code review][code-review] on
-[chat.zulip.org][czo-doc] to help review PRs and give comments on
-other people's work. Everyone is welcome to participate, even those
-new to Zulip! Even just checking out the code, manually testing it,
-and posting on whether or not it worked is valuable.
-
 [cla]: https://opensource.dropbox.com/cla/
-[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
 [dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
 [doc]: https://zulip.readthedocs.io/
-[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/code-style.html#commit-messages
 [doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
 [doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
 [doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
 [electron]: https://github.com/zulip/zulip-electron
+[gg-devel]: https://groups.google.com/forum/#!forum/zulip-devel
 [gh-issues]: https://github.com/zulip/zulip/issues
 [desktop]: https://github.com/zulip/zulip-desktop
 [android]: https://github.com/zulip/zulip-android
-[mobile]: https://github.com/zulip/zulip-mobile
-[mobile-strategy]: https://github.com/zulip/zulip-android/blob/master/android-strategy.md
+[ios]: https://github.com/zulip/zulip-ios
+[ios-exp]: https://github.com/zulip/zulip-mobile
 [email-android]: https://groups.google.com/forum/#!forum/zulip-android
 [email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
 [hubot-adapter]: https://github.com/zulip/hubot-zulip
 [jenkins]: https://github.com/zulip/zulip-jenkins-plugin
 [node]: https://github.com/zulip/zulip-node
-[zulip-js]: https://github.com/zulip/zulip-js
 [phab]: https://github.com/zulip/phabricator-to-zulip
 [puppet]: https://github.com/matthewbarr/puppet-zulip
 [redmine]: https://github.com/zulip/zulip-redmine-plugin
@@ -163,21 +129,11 @@ and posting on whether or not it worked is valuable.
 [tsearch]: https://github.com/zulip/tsearch_extras
 [transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
 [z-org]: https://github.com/zulip/zulip.github.io
-[code-review]: https://chat.zulip.org/#narrow/stream/code.20review
-[bug-report]: http://zulip.readthedocs.io/en/latest/bug-reports.html
-
-## Google Summer of Code
-
-We participated in
-[GSoC](https://developers.google.com/open-source/gsoc/) in 2016 (with
-[great results](https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/))
-and [are participating](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
-in 2017 as well.

 ## How to get involved with contributing to Zulip

 First, subscribe to the Zulip [development discussion mailing
-list](#community).
+list][gg-devel].

 The Zulip project uses a system of labels in our [issue
 tracker][gh-issues] to make it easy to find a project if you don't
@@ -185,17 +141,17 @@ have your own project idea in mind or want to get some experience with
 working on Zulip before embarking on a larger project you have in
 mind:

-* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+* [Integrations](https://github.com/zulip/zulip/labels/integrations).
  Integrate Zulip with another piece of software and contribute it
  back to the community!  Writing an integration can be a great first
  contribution.  There's detailed documentation on how to write
  integrations in [the Zulip integration writing
  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).

-* [Good first issue](https://github.com/zulip/zulip/labels/good%20first%20issue):
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
  Smaller projects that might be a great first contribution.

-* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+* [Documentation](https://github.com/zulip/zulip/labels/documentation):
  The Zulip project loves contributions of new documentation.

 * [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
@@ -211,32 +167,14 @@ mind:
  Browsing this list can be a great way to find feature ideas to
  implement that other Zulip users are excited about.

-* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
-  The projects that are
-  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
-  These are great projects if you're looking to make an impact.
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html): The
+  projects that are [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).  These are great projects if you're looking to make an impact.

-Another way to find issues in Zulip is to take advantage of our
-`area:<foo>` convention in separating out issues.  We partition all of
-our issues into areas like admin, compose, emoji, hotkeys, i18n,
-onboarding, search, etc.  Look through our
-[list of labels](https://github.com/zulip/zulip/labels), and click on
-some of the `area:` labels to see all the tickets related to your
-areas of interest.
-
-If you're excited about helping with an open issue, make sure to claim
-the issue by commenting the following in the comment section:
-"**@zulipbot** claim". **@zulipbot** will assign you to the issue and
-label the issue as **in progress**. For more details, check out
-[**@zulipbot**](https://github.com/zulip/zulipbot).
-
-You're encouraged to ask questions on how to best implement or debug
-your changes -- the Zulip maintainers are excited to answer questions
-to help you stay unblocked and working efficiently. It's great to ask
-questions in comments on GitHub issues and pull requests, or
-[on chat.zulip.org][czo-doc].  We'll direct longer discussions to
-Zulip chat, but please post a summary of what you learned from the
-chat, or link to the conversation, in a comment on the GitHub issue.
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.

 We also welcome suggestions of features that you feel would be
 valuable or changes that you feel would make Zulip a better open
@@ -254,31 +192,21 @@ responsive so this usually means we missed your message.

 For significant changes to the visual design, user experience, data
 model, or architecture, we highly recommend posting a mockup,
-screenshot, or description of what you have in mind to the
-[#design](https://chat.zulip.org/#narrow/stream/design) stream on
-[chat.zulip.org][czo-doc] to get broad feedback before you spend too
-much time on implementation details.
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.

 Finally, before implementing a larger feature, we highly recommend
-looking at the
-[new feature tutorial](http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html)
-and [coding style guidelines](http://zulip.readthedocs.io/en/latest/code-style.html)
-on ReadTheDocs.
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.

 Feedback on how to make this development process more efficient, fun,
 and friendly to new contributors is very welcome!  Just send an email
-to the [zulip-devel](#community) list with your thoughts.
-
-When you feel like you have completed your work on an issue, post your
-PR to the
-[#code review](https://chat.zulip.org/#narrow/stream/code.20review)
-stream on [chat.zulip.org][czo-doc].  This is our lightweight process
-that gives other developers the opportunity to give you comments and
-suggestions on your work.
+to the Zulip Developers list with your thoughts.

 ## License

-Copyright 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright 2011-2015 Dropbox, Inc.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -295,7 +223,4 @@ limitations under the License.
 The software includes some works released by third parties under other
 free and open source licenses. Those works are redistributed under the
 license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.
-
-
-[czo-doc]: https://zulip.readthedocs.io/en/latest/chat-zulip-org.html
+see the ``THIRDPARTY`` file included with this distribution.
--- a/docs/THIRDPARTY
+++ b/docs/THIRDPARTY
@@ -17,17 +17,27 @@ Comment:
 information or errors found in these notices.

 Files: *
-Copyright: 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright: 2011-2015 Dropbox, Inc.
 License: Apache-2

+Files: api/*
+Copyright: 2012-2014 Dropbox, Inc
+License: Expat
+
+Files: api/integrations/perforce/git_p4.py
+Copyright: 2007 Simon Hausmann <simon@lst.de>,
+ 2007 Trolltech ASA
+License: Expat
+Comment: https://raw.github.com/git/git/34022ba/git-p4.py
+
+Files: bots/jabber_mirror_backend.py
+Copyright: 2013 Permabit, Inc., 2013-2014 Dropbox, Inc.
+License: Expat
+
 Files: confirmation/*
 Copyright: 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 License: BSD-3-Clause

-Files: docs/code-of-conduct.md
-Copyright: 2017, Kandra Labs Inc.
-License: CC-BY-SA-4.0
-
 Files: puppet/apt/*
 Copyright: 2011, Evolving Web Inc.
 License: Expat
@@ -64,15 +74,19 @@ Copyright: 2003-2009 Edgewall Software
 2003-2004 Jonas Borgström <jonas@edgewall.com>
 License: BSD-3-Clause

+Files: puppet/zulip_internal/files/pagerduty_nagios.pl
+Copyright: 2011, PagerDuty, Inc.
+License: BSD-3-Clause
+
 Files: puppet/zulip_internal/files/zulip-ec2-configure-interfaces
-Copyright: 2013-2017, Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright: 2013, Dropbox, Inc.
 License: Expat

 Files: static/audio/zulip.*
 Copyright: 2011 Vidsyn
 License: CC-0-1.0

-Files: static/third/thirdparty-fonts.css
+Files: static/styles/thirdparty-fonts.css
 Copyright: 2012-2013 Dave Gandy
 License: Expat

@@ -101,9 +115,10 @@ Copyright: 2013 Nijiko Yonskai
 License: Apache-2.0
 Comment: The software has been modified.

-Files: static/generated/emoji/images/emoji/unicode/* tools/setup/emoji/*.ttf
+Files: static/third/gemoji/images/emoji/unicode/* tools/setup/emoji_dump/*.ttf
 Copyright: Google, Inc.
 License: Apache-2.0
+Comment: These are actually Noto Emoji, not gemoji.

 Files: static/third/html5-formdata/formdata.js
 Copyright: 2010 François de Metz
@@ -136,10 +151,22 @@ Copyright: 2011-2013 Henrique Boaventura
 License: Expat
 Comment: The software has been modified.

+Files: static/third/jquery-mousewheel/jquery.mousewheel.js
+Copyright: 2011 Brandon Aaron
+License: Expat
+
+Files: static/third/jquery-perfect-scrollbar/*
+Copyright: 2012 HyeonJe Jun
+License: Expat
+
 Files: static/third/jquery-throttle-debounce/*
 Copyright: 2010 "Cowboy" Ben Alman
 License: Expat or GPL

+Files: static/third/jquery-validate/*
+Copyright: 2006 - 2011 Jörn Zaefferer
+License: Expat
+
 Files: src/zulip/static/third/lazyload/*
 Copyright: 2011 Ryan Grove
 License: Expat
@@ -148,6 +175,10 @@ Files: static/third/marked/*
 Copyright: 2011-2013, Christopher Jeffrey
 License: Expat

+Files: static/third/string-prototype-codepointat/*
+Copyright: 2014 Mathias Bynens
+License: Expat
+
 Files: static/third/sockjs/sockjs-0.3.4.js
 Copyright: 2011-2012 VMware, Inc.
 2012 Douglas Crockford
@@ -165,11 +196,41 @@ Files: static/third/spectrum/*
 Copyright: 2013 Brian Grinstead
 License: Expat

+Files: static/third/spin/spin.js
+Copyright: 2011-2013 Felix Gnass
+License: Expat
+
+Files: static/third/underscore/underscore.js
+Copyright: 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+License: Expat
+Comment: https://github.com/jashkenas/underscore/blob/master/LICENSE
+
+Files: static/third/winchan/*
+Copyright: 2012 Lloyd Hilaiel
+License: Expat
+Comment: https://github.com/mozilla/winchan
+
+Files: static/third/xdate/*
+Copyright: 2010 C. F., Wong
+License: Expat
+
 Files: static/third/zocial/*
 Copyright: Sam Collins
 License: Expat

-Files: zerver/lib/bugdown/fenced_code.py
+Files: tools/inject-messages/othello
+Copyright: Shakespeare
+License: public-domain
+
+Files: tools/jslint/jslint.js
+Copyright: 2002 Douglas Crockford
+License: XXX-good-not-evil
+
+Files: tools/review
+Copyright: 2010 Ksplice, Inc.
+License: Apache-2.0
+
+Files: zerver/lib/bugdown/codehilite.py zerver/lib/bugdown/fenced_code.py
 Copyright: 2006-2008 Waylan Limberg
 License: BSD-3-Clause
 Comment: https://pypi.python.org/pypi/Markdown
@@ -182,6 +243,16 @@ Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/
 Copyright: Django Software Foundation and individual contributors
 License: BSD-3-Clause

+Files: frontend_tests/casperjs/*
+Copyright: 2011-2012 Nicolas Perriault
+ Joyent, Inc. and other Node contributors
+License: Expat
+
+Files: frontend_tests/casperjs/modules/vendors/*
+Copyright: 2011, Jeremy Ashkenas
+License: Expat
+
+
 License: Apache-2.0
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
--- a/61
+++ b/61
@@ -7,18 +7,6 @@ def command?(name)
  $?.success?
 end

-if Vagrant::VERSION == "1.8.7" then
-    path = `which curl`
-    if path.include?('/opt/vagrant/embedded/bin/curl') then
-        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
-             "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
-             "issue before provisioning. See "\
-             "https://github.com/mitchellh/vagrant/issues/7997 "\
-             "for reference."
-        exit
-    end
-end
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

  # For LXC. VirtualBox hosts use a different box, described below.
@@ -26,8 +14,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

  # The Zulip development environment runs on 9991 on the guest.
  host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
-  host_ip_addr = "127.0.0.1"
+  http_proxy = https_proxy = no_proxy = ""

  config.vm.synced_folder ".", "/vagrant", disabled: true
  config.vm.synced_folder ".", "/srv/zulip"
@@ -43,31 +30,24 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
      when "HTTPS_PROXY"; https_proxy = value
      when "NO_PROXY"; no_proxy = value
      when "HOST_PORT"; host_port = value.to_i
-      when "HOST_IP_ADDR"; host_ip_addr = value
      end
    end
  end

+  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: "127.0.0.1"
+
  if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
+    if http_proxy != ""
      config.proxy.http = http_proxy
    end
-    if !https_proxy.nil?
+    if https_proxy != ""
      config.proxy.https = https_proxy
    end
-    if !no_proxy.nil?
+    if https_proxy != ""
      config.proxy.no_proxy = no_proxy
    end
-  elsif !http_proxy.nil? or !https_proxy.nil?
-    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
-    # We haven't figured out a workaround.
-    puts 'You have specified value for proxy in ~/.zulip-vagrant-config file but did not ' \
-         'install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install ' \
-         'vagrant-proxyconf` in a terminal.  This error will appear twice.'
-    exit
  end

-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
  # Specify LXC provider before VirtualBox provider so it's preferred.
  config.vm.provider "lxc" do |lxc|
    if command? "lxc-ls"
@@ -87,43 +67,18 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
    override.vm.box = "ubuntu/trusty64"
    # It's possible we can get away with just 1.5GB; more testing needed
    vb.memory = 2048
-    vb.cpus = 2
-  end
-
-  config.vm.provider "vmware_fusion" do |vb, override|
-    override.vm.box = "puphpet/ubuntu1404-x64"
-    vb.vmx["memsize"] = "2048"
-    vb.vmx["numvcpus"] = "2"
  end

 $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
-# If the host is running SELinux remount the /sys/fs/selinux directory as read only,
-# needed for apt-get to work.
-if [ -d "/sys/fs/selinux" ]; then
-  sudo mount -o remount,ro /sys/fs/selinux
-fi
-
-# Set default locale, this prevents errors if the user has another locale set.
-if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
-    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
-fi
-
-# Provision the development environment
 ln -nsf /srv/zulip ~/zulip
-/srv/zulip/tools/provision
-
-# Run any custom provision hooks the user has configured
-if [ -f /srv/zulip/tools/custom_provision ]; then
-  chmod +x /srv/zulip/tools/custom_provision
-  /srv/zulip/tools/custom_provision
-fi
+/usr/bin/python /srv/zulip/tools/provision.py | sudo tee -a /var/log/zulip_provision.log
 SCRIPT

  config.vm.provision "shell",
-    # We want provision to be run with the permissions of the vagrant user.
+    # We want provision.py to be run with the permissions of the vagrant user.
    privileged: false,
    inline: $provision_script
 end
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -1,534 +0,0 @@
-from django.conf import settings
-from django.db import connection, models
-from django.db.models import F
-
-from analytics.models import InstallationCount, RealmCount, \
-    UserCount, StreamCount, BaseCount, FillState, Anomaly, installation_epoch, \
-    last_successful_fill
-from zerver.models import Realm, UserProfile, Message, Stream, \
-    UserActivityInterval, RealmAuditLog, models
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, ceiling_to_day, \
-    ceiling_to_hour, verify_UTC
-
-from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, Union
-
-from collections import defaultdict, OrderedDict
-from datetime import timedelta, datetime
-from zerver.lib.logging_util import create_logger
-import time
-
-## Logging setup ##
-
-logger = create_logger('zulip.management', settings.ANALYTICS_LOG_PATH, 'INFO')
-
-# You can't subtract timedelta.max from a datetime, so use this instead
-TIMEDELTA_MAX = timedelta(days=365*1000)
-
-## Class definitions ##
-
-class CountStat(object):
-    HOUR = 'hour'
-    DAY = 'day'
-    FREQUENCIES = frozenset([HOUR, DAY])
-
-    def __init__(self, property, data_collector, frequency, interval=None):
-        # type: (str, DataCollector, str, Optional[timedelta]) -> None
-        self.property = property
-        self.data_collector = data_collector
-        # might have to do something different for bitfields
-        if frequency not in self.FREQUENCIES:
-            raise AssertionError("Unknown frequency: %s" % (frequency,))
-        self.frequency = frequency
-        if interval is not None:
-            self.interval = interval
-        elif frequency == CountStat.HOUR:
-            self.interval = timedelta(hours=1)
-        else:  # frequency == CountStat.DAY
-            self.interval = timedelta(days=1)
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<CountStat: %s>" % (self.property,)
-
-class LoggingCountStat(CountStat):
-    def __init__(self, property, output_table, frequency):
-        # type: (str, Type[BaseCount], str) -> None
-        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
-
-class DependentCountStat(CountStat):
-    def __init__(self, property, data_collector, frequency, interval=None, dependencies=[]):
-        # type: (str, DataCollector, str, Optional[timedelta], List[str]) -> None
-        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
-        self.dependencies = dependencies
-
-class DataCollector(object):
-    def __init__(self, output_table, pull_function):
-        # type: (Type[BaseCount], Optional[Callable[[str, datetime, datetime], int]]) -> None
-        self.output_table = output_table
-        self.pull_function = pull_function
-
-## CountStat-level operations ##
-
-def process_count_stat(stat, fill_to_time):
-    # type: (CountStat, datetime) -> None
-    if stat.frequency == CountStat.HOUR:
-        time_increment = timedelta(hours=1)
-    elif stat.frequency == CountStat.DAY:
-        time_increment = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
-
-    verify_UTC(fill_to_time)
-    if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
-
-    fill_state = FillState.objects.filter(property=stat.property).first()
-    if fill_state is None:
-        currently_filled = installation_epoch()
-        fill_state = FillState.objects.create(property=stat.property,
-                                              end_time=currently_filled,
-                                              state=FillState.DONE)
-        logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
-    elif fill_state.state == FillState.STARTED:
-        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
-        do_delete_counts_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - time_increment
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        logger.info("UNDO DONE %s" % (stat.property,))
-    elif fill_state.state == FillState.DONE:
-        currently_filled = fill_state.end_time
-    else:
-        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
-
-    if isinstance(stat, DependentCountStat):
-        for dependency in stat.dependencies:
-            dependency_fill_time = last_successful_fill(dependency)
-            if dependency_fill_time is None:
-                logger.warning("DependentCountStat %s run before dependency %s." %
-                               (stat.property, dependency))
-                return
-            fill_to_time = min(fill_to_time, dependency_fill_time)
-
-    currently_filled = currently_filled + time_increment
-    while currently_filled <= fill_to_time:
-        logger.info("START %s %s" % (stat.property, currently_filled))
-        start = time.time()
-        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
-        do_fill_count_stat_at_hour(stat, currently_filled)
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        end = time.time()
-        currently_filled = currently_filled + time_increment
-        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
-
-def do_update_fill_state(fill_state, end_time, state):
-    # type: (FillState, datetime, int) -> None
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
-
-# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is timezone aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    start_time = end_time - stat.interval
-    if not isinstance(stat, LoggingCountStat):
-        timer = time.time()
-        assert(stat.data_collector.pull_function is not None)
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
-        logger.info("%s run pull_function (%dms/%sr)" %
-                    (stat.property, (time.time()-timer)*1000, rows_added))
-    do_aggregate_to_summary_table(stat, end_time)
-
-def do_delete_counts_at_hour(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    if isinstance(stat, LoggingCountStat):
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.output_table in [UserCount, StreamCount]:
-            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-    else:
-        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-
-def do_aggregate_to_summary_table(stat, end_time):
-    # type: (CountStat, datetime) -> None
-    cursor = connection.cursor()
-
-    # Aggregate into RealmCount
-    output_table = stat.data_collector.output_table
-    if output_table in (UserCount, StreamCount):
-        realmcount_query = """
-            INSERT INTO analytics_realmcount
-                (realm_id, value, property, subgroup, end_time)
-            SELECT
-                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
-                %(output_table)s.subgroup, %%(end_time)s
-            FROM zerver_realm
-            JOIN %(output_table)s
-            ON
-                zerver_realm.id = %(output_table)s.realm_id
-            WHERE
-                %(output_table)s.property = '%(property)s' AND
-                %(output_table)s.end_time = %%(end_time)s
-            GROUP BY zerver_realm.id, %(output_table)s.subgroup
-        """ % {'output_table': output_table._meta.db_table,
-               'property': stat.property}
-        start = time.time()
-        cursor.execute(realmcount_query, {'end_time': end_time})
-        end = time.time()
-        logger.info("%s RealmCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
-
-    # Aggregate into InstallationCount
-    installationcount_query = """
-        INSERT INTO analytics_installationcount
-            (value, property, subgroup, end_time)
-        SELECT
-            sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
-        FROM analytics_realmcount
-        WHERE
-            property = '%(property)s' AND
-            end_time = %%(end_time)s
-        GROUP BY analytics_realmcount.subgroup
-    """ % {'property': stat.property}
-    start = time.time()
-    cursor.execute(installationcount_query, {'end_time': end_time})
-    end = time.time()
-    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
-    cursor.close()
-
-## Utility functions called from outside counts.py ##
-
-# called from zerver/lib/actions.py; should not throw any errors
-def do_increment_logging_stat(zerver_object, stat, subgroup, event_time, increment=1):
-    # type: (Union[Realm, UserProfile, Stream], CountStat, Optional[Union[str, int, bool]], datetime, int) -> None
-    table = stat.data_collector.output_table
-    if table == RealmCount:
-        id_args = {'realm': zerver_object}
-    elif table == UserCount:
-        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
-    else:  # StreamCount
-        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
-
-    if stat.frequency == CountStat.DAY:
-        end_time = ceiling_to_day(event_time)
-    else:  # CountStat.HOUR:
-        end_time = ceiling_to_hour(event_time)
-
-    row, created = table.objects.get_or_create(
-        property=stat.property, subgroup=subgroup, end_time=end_time,
-        defaults={'value': increment}, **id_args)
-    if not created:
-        row.value = F('value') + increment
-        row.save(update_fields=['value'])
-
-def do_drop_all_analytics_tables():
-    # type: () -> None
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-    Anomaly.objects.all().delete()
-
-def do_drop_single_stat(property):
-    # type: (str) -> None
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-## DataCollector-level operations ##
-
-def do_pull_by_sql_query(property, start_time, end_time, query, group_by):
-    # type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int
-    if group_by is None:
-        subgroup = 'NULL'
-        group_by_clause  = ''
-    else:
-        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
-        group_by_clause = ', ' + subgroup
-
-    # We do string replacement here because cursor.execute will reject a
-    # group_by_clause given as a param.
-    # We pass in the datetimes as params to cursor.execute so that we don't have to
-    # think about how to convert python datetimes to SQL datetimes.
-    query_ = query % {'property': property, 'subgroup': subgroup,
-                      'group_by_clause': group_by_clause}
-    cursor = connection.cursor()
-    cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
-    rowcount = cursor.rowcount
-    cursor.close()
-    return rowcount
-
-def sql_data_collector(output_table, query, group_by):
-    # type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector
-    def pull_function(property, start_time, end_time):
-        # type: (str, datetime, datetime) -> int
-        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
-    return DataCollector(output_table, pull_function)
-
-def do_pull_minutes_active(property, start_time, end_time):
-    # type: (str, datetime, datetime) -> int
-    user_activity_intervals = UserActivityInterval.objects.filter(
-        end__gt=start_time, start__lt=end_time
-    ).select_related(
-        'user_profile'
-    ).values_list(
-        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
-
-    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
-    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        start = max(start_time, interval_start)
-        end = min(end_time, interval_end)
-        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
-
-    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
-                      end_time=end_time, value=int(seconds // 60))
-            for ids, seconds in seconds_active.items() if seconds >= 60]
-    UserCount.objects.bulk_create(rows)
-    return len(rows)
-
-count_message_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_message
-    ON
-        zerver_userprofile.id = zerver_message.sender_id
-    WHERE
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-# Note: ignores the group_by / group_by_clause.
-count_message_type_by_user_query = """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type = 1 THEN 'private_message'
-             WHEN
-                  zerver_recipient.type = 3 THEN 'huddle_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
-
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.pub_date >= %%(time_start)s AND
-            zerver_message.pub_date < %%(time_end)s
-        JOIN zerver_recipient
-        ON
-            zerver_message.recipient_id = zerver_recipient.id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_recipient.type_id = zerver_stream.id
-        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-
-# This query joins to the UserProfile table since all current queries that
-# use this also subgroup on UserProfile.is_bot. If in the future there is a
-# stat that counts messages by stream and doesn't need the UserProfile
-# table, consider writing a new query for efficiency.
-count_message_by_stream_query = """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-        zerver_stream.id = zerver_recipient.type_id
-    JOIN zerver_message
-    ON
-        zerver_recipient.id = zerver_message.recipient_id
-    JOIN zerver_userprofile
-    ON
-        zerver_message.sender_id = zerver_userprofile.id
-    WHERE
-        zerver_stream.date_created < %%(time_end)s AND
-        zerver_recipient.type = 2 AND
-        zerver_message.pub_date >= %%(time_start)s AND
-        zerver_message.pub_date < %%(time_end)s
-    GROUP BY zerver_stream.id %(group_by_clause)s
-"""
-
-# Hardcodes the query needed by active_users:is_bot:day, since that is
-# currently the only stat that uses this.
-count_user_by_realm_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realm
-    JOIN zerver_userprofile
-    ON
-        zerver_realm.id = zerver_userprofile.realm_id
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
-        zerver_userprofile.date_joined >= %%(time_start)s AND
-        zerver_userprofile.date_joined < %%(time_end)s AND
-        zerver_userprofile.is_active = TRUE
-    GROUP BY zerver_realm.id %(group_by_clause)s
-"""
-
-# Currently hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in ['user_created', 'user_deactivated', etc].
-# In particular, it's important to ensure that migrations don't cause that to happen.
-check_realmauditlog_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realmauditlog ral1
-    JOIN (
-        SELECT modified_user_id, max(event_time) AS max_event_time
-        FROM zerver_realmauditlog
-        WHERE
-            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
-            event_time < %%(time_end)s
-        GROUP BY modified_user_id
-    ) ral2
-    ON
-        ral1.event_time = max_event_time AND
-        ral1.modified_user_id = ral2.modified_user_id
-    JOIN zerver_userprofile
-    ON
-        ral1.modified_user_id = zerver_userprofile.id
-    WHERE
-        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
-"""
-
-check_useractivityinterval_by_user_query = """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_useractivityinterval
-    ON
-        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
-    WHERE
-        zerver_useractivityinterval.end >= %%(time_start)s AND
-        zerver_useractivityinterval.start < %%(time_end)s
-    GROUP BY zerver_userprofile.id %(group_by_clause)s
-"""
-
-count_realm_active_humans_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
-    FROM (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = 'active_users_audit:is_bot:day' AND
-            subgroup = 'false' AND
-            end_time = %%(time_end)s
-    ) usercount1
-    JOIN (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = '15day_actives::day' AND
-            end_time = %%(time_end)s
-    ) usercount2
-    ON
-        usercount1.user_id = usercount2.user_id
-    GROUP BY usercount1.realm_id
-"""
-
-# Currently unused and untested
-count_stream_by_realm_query = """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
-    FROM zerver_realm
-    JOIN zerver_stream
-    ON
-        zerver_realm.id = zerver_stream.realm_id AND
-    WHERE
-        zerver_realm.date_created < %%(time_end)s AND
-        zerver_stream.date_created >= %%(time_start)s AND
-        zerver_stream.date_created < %%(time_end)s
-    GROUP BY zerver_realm.id %(group_by_clause)s
-"""
-
-## CountStat declarations ##
-
-count_stats_ = [
-    # Messages Sent stats
-    # Stats that count the number of messages sent in various ways.
-    # These are also the set of stats that read from the Message table.
-
-    CountStat('messages_sent:is_bot:hour',
-              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.HOUR),
-    CountStat('messages_sent:message_type:day',
-              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
-    CountStat('messages_sent:client:day',
-              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
-              CountStat.DAY),
-    CountStat('messages_in_stream:is_bot:day',
-              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-
-    # Number of Users stats
-    # Stats that count the number of active users in the UserProfile.is_active sense.
-
-    # 'active_users_audit:is_bot:day' is the canonical record of which users were
-    # active on which days (in the UserProfile.is_active sense).
-    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-    CountStat('active_users_audit:is_bot:day',
-              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
-              CountStat.DAY),
-    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-    # sum sequence of 'active_users_log:is_bot:day', for any realm that
-    # started after the latter stat was introduced.
-    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
-    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-    # approximation, e.g. if a user is deactivated between the end of the
-    # day and when this stat is run, they won't be counted. However, is the
-    # simplest of the three to inspect by hand.
-    CountStat('active_users:is_bot:day',
-              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
-              CountStat.DAY, interval=TIMEDELTA_MAX),
-
-    # User Activity stats
-    # Stats that measure user activity in the UserActivityInterval sense.
-
-    CountStat('15day_actives::day',
-              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
-              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
-
-    # Dependent stats
-    # Must come after their dependencies.
-
-    # Canonical account of the number of active humans in a realm on each day.
-    DependentCountStat('realm_active_humans::day',
-                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
-                       CountStat.DAY,
-                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
-]
-
-COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
--- a/analytics/lib/fixtures.py
+++ b/analytics/lib/fixtures.py
@@ -1,68 +0,0 @@
-from zerver.models import Realm, UserProfile, Stream, Message
-from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
-from analytics.lib.counts import CountStat
-from analytics.lib.time_utils import time_range
-
-from datetime import datetime
-from math import sqrt
-from random import gauss, random, seed
-from typing import List
-
-from six.moves import zip
-
-def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
-                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
-                              frequency=CountStat.DAY, partial_sum=False, random_seed=26):
-    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
-    """
-    Generate semi-realistic looking time series data for testing analytics graphs.
-
-    days -- Number of days of data. Is the number of data points generated if
-        frequency is CountStat.DAY.
-    business_hours_base -- Average value during a business hour (or day) at beginning of
-        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
-    non_business_hours_base -- The above, for non-business hours/days.
-    growth -- Ratio between average values at end of time series and beginning of time series.
-    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
-        point is unaffected by the previous point, and at 1 each point is a deterministic
-        function of the previous point.
-    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
-        the variance.
-    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
-    frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
-    random_seed -- Seed for random number generator.
-    """
-    if frequency == CountStat.HOUR:
-        length = days*24
-        seasonality = [non_business_hours_base] * 24 * 7
-        for day in range(5):
-            for hour in range(8):
-                seasonality[24*day + hour] = business_hours_base
-        holidays  = []
-        for i in range(days):
-            holidays.extend([random() < holiday_rate] * 24)
-    elif frequency == CountStat.DAY:
-        length = days
-        seasonality = [8*business_hours_base + 16*non_business_hours_base] * 5 + \
-                      [24*non_business_hours_base] * 2
-        holidays = [random() < holiday_rate for i in range(days)]
-    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
-    if length < 2:
-        raise AssertionError("Must be generating at least 2 data points. "
-                             "Currently generating %s" % (length,))
-    growth_base = growth ** (1. / (length-1))
-    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
-
-    seed(random_seed)
-    noise_scalars = [gauss(0, 1)]
-    for i in range(1, length):
-        noise_scalars.append(noise_scalars[-1]*autocorrelation + gauss(0, 1)*(1-autocorrelation))
-
-    values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
-              for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-    if partial_sum:
-        for i in range(1, length):
-            values[i] = values[i-1] + values[i]
-    return [max(v, 0) for v in values]
--- a/analytics/lib/time_utils.py
+++ b/analytics/lib/time_utils.py
@@ -1,32 +0,0 @@
-from zerver.lib.timestamp import floor_to_hour, floor_to_day, \
-    timestamp_to_datetime, verify_UTC
-from analytics.lib.counts import CountStat
-
-from datetime import datetime, timedelta
-from typing import List, Optional
-
-# If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
-# If min_length is greater than 0, pads the list to the left.
-# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
-# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(start, end, frequency, min_length):
-    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
-    verify_UTC(start)
-    verify_UTC(end)
-    if frequency == CountStat.HOUR:
-        end = floor_to_hour(end)
-        step = timedelta(hours=1)
-    elif frequency == CountStat.DAY:
-        end = floor_to_day(end)
-        step = timedelta(days=1)
-    else:
-        raise AssertionError("Unknown frequency: %s" % (frequency,))
-
-    times = []
-    if min_length is not None:
-        start = min(start, end - (min_length-1)*step)
-    current = end
-    while current >= start:
-        times.append(current)
-        current -= step
-    return list(reversed(times))
--- a/analytics/management/commands/active_user_stats.py
+++ b/analytics/management/commands/active_user_stats.py
@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from typing import Any
+
+from zerver.models import UserPresence, UserActivity
+from zerver.lib.utils import statsd, statsd_key
+
+from datetime import datetime, timedelta
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = """Sends active user statistics to statsd.
+
+    Run as a cron job that runs every 10 minutes."""
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        # Get list of all active users in the last 1 week
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+
+        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
+
+        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
+        hour_buckets = [0.16, 2, 12, 24, 48, 168]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+
+        for last_presence in users:
+            if last_presence.status == UserPresence.IDLE:
+                known_active = last_presence.timestamp - timedelta(minutes=30)
+            else:
+                known_active = last_presence.timestamp
+
+            for bucket in hour_buckets:
+                if bucket not in user_info[last_presence.user_profile.realm.domain]:
+                    user_info[last_presence.user_profile.realm.domain][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
+
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
+
+        # Also do stats for how many users have been reading the app.
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
+        user_info = defaultdict(dict)
+        for activity in users_reading:
+            for bucket in hour_buckets:
+                if bucket not in user_info[activity.user_profile.realm.domain]:
+                    user_info[activity.user_profile.realm.domain][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers reading for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
--- a/analytics/management/commands/active_user_stats_by_day.py
+++ b/analytics/management/commands/active_user_stats_by_day.py
@@ -0,0 +1,27 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import datetime
+import pytz
+
+from optparse import make_option
+from typing import Any
+from django.core.management.base import BaseCommand
+from zerver.lib.statistics import activity_averages_during_day
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity for a given day."
+
+    option_list = BaseCommand.option_list + \
+        (make_option('--date', default=None, action='store',
+                     help="Day to query in format 2013-12-05.  Default is yesterday"),)
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options["date"] is None:
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")
--- a/analytics/management/commands/analyze_mit.py
+++ b/analytics/management/commands/analyze_mit.py
@@ -1,6 +1,10 @@
-from typing import Any, Dict
+from __future__ import absolute_import
+from __future__ import print_function

-from django.core.management.base import BaseCommand, CommandParser
+from typing import Any
+
+from optparse import make_option
+from django.core.management.base import BaseCommand
 from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
 import datetime
@@ -13,7 +17,7 @@ def compute_stats(log_level):
    logger.setLevel(log_level)

    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
+    mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
                                       recipient__type=Recipient.STREAM,
                                       pub_date__gt=one_week_ago)
    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
@@ -26,15 +30,15 @@ def compute_stats(log_level):
                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                           "root@mit.edu", "nagios@mit.edu",
                           "www-data|local-realm@mit.edu"])
-    user_counts = {}  # type: Dict[str, Dict[str, int]]
+    user_counts = {} # type: Dict[str, Dict[str, int]]
    for m in mit_query.select_related("sending_client", "sender"):
        email = m.sender.email
        user_counts.setdefault(email, {})
        user_counts[email].setdefault(m.sending_client.name, 0)
        user_counts[email][m.sending_client.name] += 1

-    total_counts = {}  # type: Dict[str, int]
-    total_user_counts = {}  # type: Dict[str, int]
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
    for email, counts in user_counts.items():
        total_user_counts.setdefault(email, 0)
        for client_name, count in counts.items():
@@ -43,13 +47,13 @@ def compute_stats(log_level):
            total_user_counts[email] += count

    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}  # type: Dict[int, float]
+    top_percents = {} # type: Dict[int, float]
    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
        top_percents[size] = 0.0
    for i, email in enumerate(sorted(total_user_counts.keys(),
                                     key=lambda x: -total_user_counts[x])):
        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
-                              total_user_counts[email], 1)
+                               total_user_counts[email], 1)
        for size in top_percents.keys():
            top_percents.setdefault(size, 0)
            if i < size:
@@ -69,11 +73,10 @@ def compute_stats(log_level):
        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))

 class Command(BaseCommand):
-    help = "Compute statistics on MIT Zephyr usage."
+    option_list = BaseCommand.option_list + \
+        (make_option('--verbose', default=False, action='store_true'),)

-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
-        parser.add_argument('--verbose', default=False, action='store_true')
+    help = "Compute statistics on MIT Zephyr usage."

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
--- a/analytics/management/commands/analyze_user_activity.py
+++ b/analytics/management/commands/analyze_user_activity.py
@@ -1,8 +1,13 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 from typing import Any, Dict

 from zerver.lib.statistics import seconds_usage_between

-from django.core.management.base import BaseCommand, CommandParser
+from optparse import make_option
+from django.core.management.base import BaseCommand
 from zerver.models import UserProfile
 import datetime
 from django.utils.timezone import utc
@@ -14,7 +19,7 @@ def analyze_activity(options):

    user_profile_query = UserProfile.objects.all()
    if options["realm"]:
-        user_profile_query = user_profile_query.filter(realm__string_id=options["realm"])
+        user_profile_query = user_profile_query.filter(realm__domain=options["realm"])

    print("Per-user online duration:\n")
    total_duration = datetime.timedelta(0)
@@ -42,17 +47,17 @@ It will correctly not count server-initiated reloads in the activity statistics.

 The duration flag can be used to control how many days to show usage duration for

-Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--duration=1]
+Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]

 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""

-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
-        parser.add_argument('--realm', action='store')
-        parser.add_argument('--date', action='store', default="2013-09-06")
-        parser.add_argument('--duration', action='store', default=1, type=int,
-                            help="How many days to show usage information for")
+    option_list = BaseCommand.option_list + (
+        make_option('--realm', action='store'),
+        make_option('--date', action='store', default="2013-09-06"),
+        make_option('--duration', action='store', default=1, type=int,
+                    help="How many days to show usage information for"),
+        )

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
--- a/analytics/management/commands/clear_analytics_tables.py
+++ b/analytics/management/commands/clear_analytics_tables.py
@@ -1,26 +0,0 @@
-import sys
-
-from argparse import ArgumentParser
-from django.db import connection
-from django.core.management.base import BaseCommand
-
-from analytics.lib.counts import do_drop_all_analytics_tables
-
-from typing import Any
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser):
-        # type: (ArgumentParser) -> None
-        parser.add_argument('--force',
-                            action='store_true',
-                            help="Clear analytics tables.")
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        if options['force']:
-            do_drop_all_analytics_tables()
-        else:
-            print("Would delete all data from analytics tables (!); use --force to do so.")
-            sys.exit(1)
--- a/analytics/management/commands/clear_single_stat.py
+++ b/analytics/management/commands/clear_single_stat.py
@@ -1,33 +0,0 @@
-import sys
-
-from argparse import ArgumentParser
-from django.db import connection
-from django.core.management.base import BaseCommand
-
-from analytics.lib.counts import do_drop_single_stat, COUNT_STATS
-
-from typing import Any
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser):
-        # type: (ArgumentParser) -> None
-        parser.add_argument('--force',
-                            action='store_true',
-                            help="Actually do it.")
-        parser.add_argument('--property',
-                            type=str,
-                            help="The property of the stat to be cleared.")
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        property = options['property']
-        if property not in COUNT_STATS:
-            print("Invalid property: %s" % (property,))
-            sys.exit(1)
-        if not options['force']:
-            print("No action taken. Use --force.")
-            sys.exit(1)
-
-        do_drop_single_stat(property)
--- a/analytics/management/commands/client_activity.py
+++ b/analytics/management/commands/client_activity.py
@@ -1,32 +1,30 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 from typing import Any

 from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
-from django.utils.timezone import now as timezone_now

-from zerver.lib.management import ZulipBaseCommand
-from zerver.models import UserActivity
+from zerver.models import UserActivity, UserProfile, Realm, \
+    get_realm, get_user_profile_by_email

 import datetime

-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
    help = """Report rough client activity globally, for a realm, or for a user

 Usage examples:

-./manage.py client_activity --target server
-./manage.py client_activity --target realm --realm zulip
-./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
+python manage.py client_activity
+python manage.py client_activity zulip.com
+python manage.py client_activity jesstess@zulip.com"""

    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
-        parser.add_argument('--target', dest='target', required=True, type=str,
-                            help="'server' will calculate client activity of the entire server. "
-                                 "'realm' will calculate client activity of realm. "
-                                 "'user' will calculate client activity of the user.")
-        parser.add_argument('--user', dest='user', type=str,
-                            help="The email adress of the user you want to calculate activity.")
-        self.add_realm_args(parser)
+        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
+                            help="realm or user to estimate client activity for")

    def compute_activity(self, user_activity_objects):
        # type: (QuerySet) -> None
@@ -40,7 +38,7 @@ Usage examples:
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
-        threshold = timezone_now() - datetime.timedelta(days=7)
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))
@@ -59,19 +57,25 @@ Usage examples:
            print("%25s %15d" % (count[1], count[0]))
        print("Total:", total)

+
    def handle(self, *args, **options):
        # type: (*Any, **str) -> None
-        realm = self.get_realm(options)
-        if options["user"] is None:
-            if options["target"] == "server" and realm is None:
-                # Report global activity.
-                self.compute_activity(UserActivity.objects.all())
-            elif options["target"] == "realm" and realm is not None:
-                self.compute_activity(UserActivity.objects.filter(user_profile__realm=realm))
-            else:
-                self.print_help("./manage.py", "client_activity")
-        elif options["target"] == "user":
-            user_profile = self.get_user(options["user"], realm)
-            self.compute_activity(UserActivity.objects.filter(user_profile=user_profile))
+        if options['arg'] is None:
+            # Report global activity.
+            self.compute_activity(UserActivity.objects.all())
        else:
-            self.print_help("./manage.py", "client_activity")
+            arg = options['arg']
+            try:
+                # Report activity for a user.
+                user_profile = get_user_profile_by_email(arg)
+                self.compute_activity(UserActivity.objects.filter(
+                        user_profile=user_profile))
+            except UserProfile.DoesNotExist:
+                try:
+                    # Report activity for a realm.
+                    realm = get_realm(arg)
+                    self.compute_activity(UserActivity.objects.filter(
+                            user_profile__realm=realm))
+                except Realm.DoesNotExist:
+                    print("Unknown user or domain %s" % (arg,))
+                    exit(1)
--- a/analytics/management/commands/populate_analytics_db.py
+++ b/analytics/management/commands/populate_analytics_db.py
@@ -1,137 +0,0 @@
-
-from argparse import ArgumentParser
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
-from analytics.lib.fixtures import generate_time_series_data
-from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, FillState
-from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, UserProfile, Stream, Message, Client, \
-    RealmAuditLog
-
-from datetime import datetime, timedelta
-
-from six.moves import zip
-from typing import Any, Dict, List, Optional, Text, Type, Union, Mapping
-
-class Command(BaseCommand):
-    help = """Populates analytics tables with randomly generated data."""
-
-    DAYS_OF_DATA = 100
-    random_seed = 26
-
-    def create_user(self, email, full_name, is_staff, date_joined, realm):
-        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
-        user = UserProfile.objects.create(
-            email=email, full_name=full_name, is_staff=is_staff,
-            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
-            api_key='42', date_joined=date_joined)
-        RealmAuditLog.objects.create(
-            realm=realm, modified_user=user, event_type='user_created',
-            event_time=user.date_joined)
-        return user
-
-    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
-                              growth, autocorrelation, spikiness, holiday_rate=0,
-                              partial_sum=False):
-        # type: (CountStat, float, float, float, float, float, float, bool) -> List[int]
-        self.random_seed += 1
-        return generate_time_series_data(
-            days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
-            non_business_hours_base=non_business_hours_base, growth=growth,
-            autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        do_drop_all_analytics_tables()
-        # I believe this also deletes any objects with this realm as a foreign key
-        Realm.objects.filter(string_id='analytics').delete()
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
-        realm = Realm.objects.create(
-            string_id='analytics', name='Analytics', date_created=installation_time)
-        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
-
-        def insert_fixture_data(stat, fixture_data, table):
-            # type: (CountStat, Mapping[Optional[str], List[int]], Type[BaseCount]) -> None
-            end_times = time_range(last_end_time, last_end_time, stat.frequency,
-                                   len(list(fixture_data.values())[0]))
-            if table == RealmCount:
-                id_args = {'realm': realm}
-            if table == UserCount:
-                id_args = {'realm': realm, 'user': shylock}
-            for subgroup, values in fixture_data.items():
-                table.objects.bulk_create([
-                    table(property=stat.property, subgroup=subgroup, end_time=end_time,
-                          value=value, **id_args)
-                    for end_time, value in zip(end_times, values) if value != 0])
-
-        stat = COUNT_STATS['realm_active_humans::day']
-        realm_data = {
-            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
-        }  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data = {'false': self.generate_fixture_data(
-            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
-                      'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        user_data = {
-            'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
-            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
-            'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        website, created = Client.objects.get_or_create(name='website')
-        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
-        android, created = Client.objects.get_or_create(name='ZulipAndroid')
-        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
-        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
-        API, created = Client.objects.get_or_create(name='API: Python')
-        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
-        unused, created = Client.objects.get_or_create(name='unused')
-        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
-
-        stat = COUNT_STATS['messages_sent:client:day']
-        user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        # TODO: messages_sent_to_stream:is_bot
--- a/analytics/management/commands/realm_stats.py
+++ b/analytics/management/commands/realm_stats.py
@@ -1,4 +1,8 @@
-from typing import Any, List
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any

 from argparse import ArgumentParser
 import datetime
@@ -6,8 +10,6 @@ import pytz

 from django.core.management.base import BaseCommand
 from django.db.models import Count
-from django.utils.timezone import now as timezone_now
-
 from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
    Subscription, UserMessage, get_realm

@@ -27,27 +29,27 @@ class Command(BaseCommand):
    def active_users(self, realm):
        # type: (Realm) -> List[UserProfile]
        # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
-        return [activity.user_profile for activity in (
-            UserActivity.objects.filter(user_profile__realm=realm,
-                                        user_profile__is_active=True,
-                                        last_visit__gt=activity_cutoff,
-                                        query="/json/users/me/pointer",
-                                        client__name="website"))]
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
+        return [activity.user_profile for activity in \
+                    UserActivity.objects.filter(user_profile__realm=realm,
+                                                user_profile__is_active=True,
+                                                last_visit__gt=activity_cutoff,
+                                                query="/json/users/me/pointer",
+                                                client__name="website")]

    def messages_sent_by(self, user, days_ago):
        # type: (UserProfile, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()

    def total_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

    def human_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()

    def api_messages(self, realm, days_ago):
@@ -56,19 +58,19 @@ class Command(BaseCommand):

    def stream_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                     recipient__type=Recipient.STREAM).count()

    def private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()

    def group_private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()

@@ -84,7 +86,7 @@ class Command(BaseCommand):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -92,7 +94,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)

            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            active_users = self.active_users(realm)
@@ -119,7 +121,7 @@ class Command(BaseCommand):
                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))

-            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
+            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications == True])
            self.report_percentage(num_notifications_enabled, num_active,
                                   "active users have desktop notifications enabled")

--- a/analytics/management/commands/stream_stats.py
+++ b/analytics/management/commands/stream_stats.py
@@ -1,3 +1,6 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 from typing import Any

 from argparse import ArgumentParser
@@ -17,7 +20,7 @@ class Command(BaseCommand):
        # type: (*Any, **str) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -25,7 +28,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)
            print("------------")
            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
--- a/analytics/management/commands/update_analytics_counts.py
+++ b/analytics/management/commands/update_analytics_counts.py
@@ -1,93 +0,0 @@
-import os
-import sys
-from scripts.lib.zulip_tools import ENDC, WARNING
-
-from argparse import ArgumentParser
-from datetime import timedelta
-import time
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-from django.utils.timezone import utc as timezone_utc
-from django.utils.dateparse import parse_datetime
-from django.conf import settings
-
-from analytics.models import RealmCount, UserCount
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import UserProfile, Message, Realm
-
-from typing import Any, Dict
-
-class Command(BaseCommand):
-    help = """Fills Analytics tables.
-
-    Run as a cron job that runs every hour."""
-
-    def add_arguments(self, parser):
-        # type: (ArgumentParser) -> None
-        parser.add_argument('--time', '-t',
-                            type=str,
-                            help='Update stat tables from current state to --time. Defaults to the current time.',
-                            default=timezone_now().isoformat())
-        parser.add_argument('--utc',
-                            action='store_true',
-                            help="Interpret --time in UTC.",
-                            default=False)
-        parser.add_argument('--stat', '-s',
-                            type=str,
-                            help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument('--verbose',
-                            action='store_true',
-                            help="Print timing information to stdout.",
-                            default=False)
-
-    def handle(self, *args, **options):
-        # type: (*Any, **Any) -> None
-        try:
-            os.mkdir(settings.ANALYTICS_LOCK_DIR)
-        except OSError:
-            print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
-            return
-
-        try:
-            self.run_update_analytics_counts(options)
-        finally:
-            os.rmdir(settings.ANALYTICS_LOCK_DIR)
-
-    def run_update_analytics_counts(self, options):
-        # type: (Dict[str, Any]) -> None
-        # installation_epoch relies on there being at least one realm; we
-        # shouldn't run the analytics code if that condition isn't satisfied
-        if not Realm.objects.exists():
-            logger.info("No realms, stopping update_analytics_counts")
-            return
-
-        fill_to_time = parse_datetime(options['time'])
-        if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
-        if fill_to_time.tzinfo is None:
-            raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
-
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
-
-        if options['stat'] is not None:
-            stats = [COUNT_STATS[options['stat']]]
-        else:
-            stats = list(COUNT_STATS.values())
-
-        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
-        if options['verbose']:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options['verbose']:
-                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
-                last = time.time()
-
-        if options['verbose']:
-            print("Finished updating analytics counts through %s in %.3fs" %
-                  (fill_to_time, time.time() - start))
-        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
--- a/analytics/management/commands/user_stats.py
+++ b/analytics/management/commands/user_stats.py
@@ -1,12 +1,14 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
 from argparse import ArgumentParser
 import datetime
 import pytz
 from typing import Any

 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
 from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range

 class Command(BaseCommand):
    help = "Generate statistics on user activity."
@@ -18,15 +20,15 @@ class Command(BaseCommand):

    def messages_sent_by(self, user, week):
        # type: (UserProfile, int) -> int
-        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
-        end = timezone_now() - datetime.timedelta(days=week*7)
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()

    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
@@ -34,7 +36,7 @@ class Command(BaseCommand):
            realms = Realm.objects.all()

        for realm in realms:
-            print(realm.string_id)
+            print(realm.domain)
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            print("%d users" % (len(user_profiles),))
            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
--- a/analytics/migrations/0001_initial.py
+++ b/analytics/migrations/0001_initial.py
@@ -1,112 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import models, migrations
-import django.db.models.deletion
-from django.conf import settings
-import zerver.lib.str_utils
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('zerver', '0030_realm_org_type'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Anomaly',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('info', models.CharField(max_length=1000)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='HuddleCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('huddle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='InstallationCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='RealmCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
-
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='StreamCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream')),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.CreateModel(
-            name='UserCount',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-                ('property', models.CharField(max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('interval', models.CharField(max_length=20)),
-                ('value', models.BigIntegerField()),
-                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='huddlecount',
-            unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
-        ),
-    ]
--- a/analytics/migrations/0002_remove_huddlecount.py
+++ b/analytics/migrations/0002_remove_huddlecount.py
@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='huddlecount',
-            unique_together=set([]),
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='huddle',
-        ),
-        migrations.RemoveField(
-            model_name='huddlecount',
-            name='user',
-        ),
-        migrations.DeleteModel(
-            name='HuddleCount',
-        ),
-    ]
--- a/analytics/migrations/0003_fillstate.py
+++ b/analytics/migrations/0003_fillstate.py
@@ -1,24 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations, models
-import zerver.lib.str_utils
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0002_remove_huddlecount'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='FillState',
-            fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('property', models.CharField(unique=True, max_length=40)),
-                ('end_time', models.DateTimeField()),
-                ('state', models.PositiveSmallIntegerField()),
-                ('last_modified', models.DateTimeField(auto_now=True)),
-            ],
-            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
-        ),
-    ]
--- a/analytics/migrations/0004_add_subgroup.py
+++ b/analytics/migrations/0004_add_subgroup.py
@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0003_fillstate'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='installationcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='realmcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='streamcount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name='usercount',
-            name='subgroup',
-            field=models.CharField(max_length=16, null=True),
-        ),
-    ]
--- a/analytics/migrations/0005_alter_field_size.py
+++ b/analytics/migrations/0005_alter_field_size.py
@@ -1,52 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0004_add_subgroup'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='installationcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='installationcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='interval',
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='property',
-            field=models.CharField(max_length=32),
-        ),
-    ]
--- a/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
+++ b/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0005_alter_field_size'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
-        ),
-    ]
--- a/analytics/migrations/0007_remove_interval.py
+++ b/analytics/migrations/0007_remove_interval.py
@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10.4 on 2017-01-16 20:50
-from django.conf import settings
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0006_add_subgroup_to_unique_constraints'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set([('property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='installationcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='realmcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='streamcount',
-            name='interval',
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
-        ),
-        migrations.RemoveField(
-            model_name='usercount',
-            name='interval',
-        ),
-    ]
--- a/analytics/migrations/0008_add_count_indexes.py
+++ b/analytics/migrations/0008_add_count_indexes.py
@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('zerver', '0050_userprofile_avatar_version'),
-        ('analytics', '0007_remove_interval'),
-    ]
-
-    operations = [
-        migrations.AlterIndexTogether(
-            name='realmcount',
-            index_together=set([('property', 'end_time')]),
-        ),
-        migrations.AlterIndexTogether(
-            name='streamcount',
-            index_together=set([('property', 'realm', 'end_time')]),
-        ),
-        migrations.AlterIndexTogether(
-            name='usercount',
-            index_together=set([('property', 'realm', 'end_time')]),
-        ),
-    ]
--- a/analytics/migrations/0009_remove_messages_to_stream_stat.py
+++ b/analytics/migrations/0009_remove_messages_to_stream_stat.py
@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-
-def delete_messages_sent_to_stream_stat(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent_to_stream:is_bot'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0008_add_count_indexes'),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]
--- a/analytics/migrations/0010_clear_messages_sent_values.py
+++ b/analytics/migrations/0010_clear_messages_sent_values.py
@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-def clear_message_sent_by_message_type_values(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    property = 'messages_sent:message_type:day'
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]
--- a/analytics/migrations/0011_clear_analytics_tables.py
+++ b/analytics/migrations/0011_clear_analytics_tables.py
@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db import migrations
-
-
-def clear_analytics_tables(apps, schema_editor):
-    # type: (StateApps, DatabaseSchemaEditor) -> None
-    UserCount = apps.get_model('analytics', 'UserCount')
-    StreamCount = apps.get_model('analytics', 'StreamCount')
-    RealmCount = apps.get_model('analytics', 'RealmCount')
-    InstallationCount = apps.get_model('analytics', 'InstallationCount')
-    FillState = apps.get_model('analytics', 'FillState')
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0010_clear_messages_sent_values'),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]
--- a/analytics/models.py
+++ b/analytics/models.py
@@ -1,109 +0,0 @@
-from django.db import models
-
-from zerver.models import Realm, UserProfile, Stream, Recipient
-from zerver.lib.str_utils import ModelReprMixin
-from zerver.lib.timestamp import floor_to_day
-
-import datetime
-
-from typing import Optional, Tuple, Union, Dict, Any, Text
-
-class FillState(ModelReprMixin, models.Model):
-    property = models.CharField(max_length=40, unique=True)  # type: Text
-    end_time = models.DateTimeField()  # type: datetime.datetime
-
-    # Valid states are {DONE, STARTED}
-    DONE = 1
-    STARTED = 2
-    state = models.PositiveSmallIntegerField()  # type: int
-
-    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
-
-# The earliest/starting end_time in FillState
-# We assume there is at least one realm
-def installation_epoch():
-    # type: () -> datetime.datetime
-    earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    return floor_to_day(earliest_realm_creation)
-
-def last_successful_fill(property):
-    # type: (str) -> Optional[datetime.datetime]
-    fillstate = FillState.objects.filter(property=property).first()
-    if fillstate is None:
-        return None
-    if fillstate.state == FillState.DONE:
-        return fillstate.end_time
-    return fillstate.end_time - datetime.timedelta(hours=1)
-
-# would only ever make entries here by hand
-class Anomaly(ModelReprMixin, models.Model):
-    info = models.CharField(max_length=1000)  # type: Text
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<Anomaly: %s... %s>" % (self.info, self.id)
-
-class BaseCount(ModelReprMixin, models.Model):
-    # Note: When inheriting from BaseCount, you may want to rearrange
-    # the order of the columns in the migration to make sure they
-    # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)  # type: Text
-    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[Text]
-    end_time = models.DateTimeField()  # type: datetime.datetime
-    value = models.BigIntegerField()  # type: int
-    anomaly = models.ForeignKey(Anomaly, null=True)  # type: Optional[Anomaly]
-
-    class Meta(object):
-        abstract = True
-
-class InstallationCount(BaseCount):
-
-    class Meta(object):
-        unique_together = ("property", "subgroup", "end_time")
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
-
-class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("realm", "property", "subgroup", "end_time")
-        index_together = ["property", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
-
-class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile)
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("user", "property", "subgroup", "end_time")
-        # This index dramatically improves the performance of
-        # aggregating from users to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
-
-class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream)
-    realm = models.ForeignKey(Realm)
-
-    class Meta(object):
-        unique_together = ("stream", "property", "subgroup", "end_time")
-        # This index dramatically improves the performance of
-        # aggregating from streams to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __unicode__(self):
-        # type: () -> Text
-        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)
--- a/analytics/tests/test_counts.py
+++ b/analytics/tests/test_counts.py
--- a/analytics/tests/test_fixtures.py
+++ b/analytics/tests/test_fixtures.py
@@ -1,31 +0,0 @@
-from zerver.lib.test_classes import ZulipTestCase
-
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self):
-        # type: () -> None
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=1500,
-            growth=2, spikiness=0, frequency=CountStat.HOUR)
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assertEqual(len(data), 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500*2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1, business_hours_base=2000, non_business_hours_base=2000,
-            autocorrelation=1, frequency=CountStat.HOUR)
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])
--- a/analytics/tests/test_views.py
+++ b/analytics/tests/test_views.py
@@ -1,323 +0,0 @@
-from django.utils.timezone import get_fixed_timezone, utc
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, \
-    datetime_to_timestamp
-from zerver.models import Realm, UserProfile, Client, get_realm
-
-from analytics.lib.counts import CountStat, COUNT_STATS
-from analytics.lib.time_utils import time_range
-from analytics.models import RealmCount, UserCount, BaseCount, \
-    FillState, last_successful_fill
-from analytics.views import stats, get_chart_data, sort_by_totals, \
-    sort_client_labels, rewrite_client_arrays
-
-from datetime import datetime, timedelta
-import mock
-import ujson
-
-from typing import List, Dict, Optional
-
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self):
-        # type: () -> None
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        result = self.client_get('/stats')
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
-
-class TestGetChartData(ZulipTestCase):
-    def setUp(self):
-        # type: () -> None
-        self.realm = get_realm('zulip')
-        self.user = self.example_user('hamlet')
-        self.login(self.user.email)
-        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
-                               for i in range(4)]
-        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
-                              for i in range(4)]
-
-    def data(self, i):
-        # type: (int) -> List[int]
-        return [0, 0, i, 0]
-
-    def insert_data(self, stat, realm_subgroups, user_subgroups):
-        # type: (CountStat, List[Optional[str]], List[str]) -> None
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create([
-            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                       value=100+i, realm=self.realm)
-            for i, subgroup in enumerate(realm_subgroups)])
-        UserCount.objects.bulk_create([
-            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
-                      value=200+i, realm=self.realm, user=self.user)
-            for i, subgroup in enumerate(user_subgroups)])
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'human': self.data(100)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_over_time(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        self.insert_data(stat, ['true', 'false'], ['false'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-            'frequency': CountStat.HOUR,
-            'realm': {'bot': self.data(100), 'human': self.data(101)},
-            'user': {'bot': self.data(0), 'human': self.data(200)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_message_type(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        self.insert_data(stat, ['public_stream', 'private_message'],
-                         ['public_stream', 'private_stream'])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'Public streams': self.data(100), 'Private streams': self.data(0),
-                      'Private messages': self.data(101), 'Group private messages': self.data(0)},
-            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
-                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
-            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
-            'result': 'success',
-        })
-
-    def test_messages_sent_by_client(self):
-        # type: () -> None
-        stat = COUNT_STATS['messages_sent:client:day']
-        client1 = Client.objects.create(name='client 1')
-        client2 = Client.objects.create(name='client 2')
-        client3 = Client.objects.create(name='client 3')
-        client4 = Client.objects.create(name='client 4')
-        self.insert_data(stat, [client4.id, client3.id, client2.id],
-                         [client3.id, client1.id])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
-            'frequency': CountStat.DAY,
-            'realm': {'client 4': self.data(100), 'client 3': self.data(101),
-                      'client 2': self.data(102)},
-            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
-            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
-            'result': 'success',
-        })
-
-    def test_include_empty_subgroups(self):
-        # type: () -> None
-        FillState.objects.create(
-            property='realm_active_humans::day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['realm'], {'human': [0]})
-        self.assertFalse('user' in data)
-
-        FillState.objects.create(
-            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['realm'], {'human': [0], 'bot': [0]})
-        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:message_type:day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_message_type'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['realm'], {
-            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
-        self.assertEqual(data['user'], {
-            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
-
-        FillState.objects.create(
-            property='messages_sent:client:day', end_time=self.end_times_day[0], state=FillState.DONE)
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_by_client'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['realm'], {})
-        self.assertEqual(data['user'], {})
-
-    def test_start_and_end(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[1],
-                                  'end': end_time_timestamps[2]})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
-        self.assertEqual(data['realm'], {'human': [0, 100]})
-
-        # start later then end
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'start': end_time_timestamps[2],
-                                  'end': end_time_timestamps[1]})
-        self.assert_json_error_contains(result, 'Start time is later than')
-
-    def test_min_length(self):
-        # type: () -> None
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-        # test min_length is too short to change anything
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 2})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
-        self.assertEqual(data['realm'], {'human': self.data(100)})
-        # test min_length larger than filled data
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'number_of_humans',
-                                  'min_length': 5})
-        self.assert_json_success(result)
-        data = result.json()
-        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
-        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(data['realm'], {'human': [0]+self.data(100)})
-
-    def test_non_existent_chart(self):
-        # type: () -> None
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'does_not_exist'})
-        self.assert_json_error_contains(result, 'Unknown chart name')
-
-    def test_analytics_not_running(self):
-        # type: () -> None
-        # try to get data for a valid chart, but before we've put anything in the database
-        # (e.g. before update_analytics_counts has been run)
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'number_of_humans'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    # last_successful_fill is in analytics/models.py, but get_chart_data is
-    # the only function that uses it at the moment
-    def test_last_successful_fill(self):
-        # type: () -> None
-        self.assertIsNone(last_successful_fill('non-existant'))
-        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
-        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
-        fillstate = FillState.objects.create(property='property', end_time=a_time,
-                                             state=FillState.DONE)
-        self.assertEqual(last_successful_fill('property'), a_time)
-        fillstate.state = FillState.STARTED
-        fillstate.save()
-        self.assertEqual(last_successful_fill('property'), one_hour_before)
-
-    def test_sort_by_totals(self):
-        # type: () -> None
-        empty = []  # type: List[int]
-        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
-        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
-
-    def test_sort_client_labels(self):
-        # type: () -> None
-        data = {'realm': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
-                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
-        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
-
-class TestTimeRange(ZulipTestCase):
-    def test_time_range(self):
-        # type: () -> None
-        HOUR = timedelta(hours=1)
-        DAY = timedelta(days=1)
-
-        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
-        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
-        floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
-
-        # test start == end
-        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
-        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
-        # test start == end == boundary, and min_length == 0
-        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
-        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
-        # test start and end on different boundaries
-        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, None),
-                         [floor_hour, floor_hour+HOUR])
-        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, None),
-                         [floor_day, floor_day+DAY])
-        # test min_length
-        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, 4),
-                         [floor_hour-2*HOUR, floor_hour-HOUR, floor_hour, floor_hour+HOUR])
-        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, 4),
-                         [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
-
-class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self):
-        # type: () -> None
-        a = {'desktop app 1.0': [1, 2, 3],
-             'desktop app 2.0': [10, 12, 13],
-             'desktop app 3.0': [21, 22, 23],
-             'website': [1, 2, 3],
-             'ZulipiOS': [1, 2, 3],
-             'ZulipElectron': [2, 5, 7],
-             'ZulipMobile': [1, 5, 7],
-             'ZulipPython': [1, 2, 3],
-             'API: Python': [1, 2, 3],
-             'SomethingRandom': [4, 5, 6],
-             'ZulipGitHubWebhook': [7, 7, 9],
-             'ZulipAndroid': [64, 63, 65]}
-        result = rewrite_client_arrays(a)
-        self.assertEqual(result,
-                         {'Old desktop app': [32, 36, 39],
-                          'Old iOS app': [1, 2, 3],
-                          'Desktop app': [2, 5, 7],
-                          'Mobile app': [1, 5, 7],
-                          'Website': [1, 2, 3],
-                          'Python API': [2, 4, 6],
-                          'SomethingRandom': [4, 5, 6],
-                          'GitHub webhook': [7, 7, 9],
-                          'Old Android app': [64, 63, 65]})
--- a/analytics/urls.py
+++ b/analytics/urls.py
@@ -1,39 +1,9 @@
-from django.conf.urls import url, include
-from zerver.lib.rest import rest_dispatch
-
-import analytics.views
+from django.conf.urls import patterns, url

 i18n_urlpatterns = [
-    # Server admin (user_profile.is_staff) visible stats pages
-    url(r'^activity$', analytics.views.get_activity,
-        name='analytics.views.get_activity'),
-    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
-        name='analytics.views.get_realm_activity'),
-    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
-        name='analytics.views.get_user_activity'),
-
-    # User-visible stats page
-    url(r'^stats$', analytics.views.stats,
-        name='analytics.views.stats'),
+    url(r'^activity$', 'analytics.views.get_activity'),
+    url(r'^realm_activity/(?P<realm>[\S]+)/$', 'analytics.views.get_realm_activity'),
+    url(r'^user_activity/(?P<email>[\S]+)/$', 'analytics.views.get_user_activity'),
 ]

-# These endpoints are a part of the API (V1), which uses:
-# * REST verbs
-# * Basic auth (username:password is email:apiKey)
-# * Takes and returns json-formatted data
-#
-# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
-#
-# All of these paths are accessed by either a /json or /api prefix
-v1_api_and_json_patterns = [
-    # get data for the graphs at /stats
-    url(r'^analytics/chart_data$', rest_dispatch,
-        {'GET': 'analytics.views.get_chart_data'}),
-]
-
-i18n_urlpatterns += [
-    url(r'^api/v1/', include(v1_api_and_json_patterns)),
-    url(r'^json/', include(v1_api_and_json_patterns)),
-]
-
-urlpatterns = i18n_urlpatterns
+urlpatterns = patterns('', *i18n_urlpatterns)
--- a/analytics/views.py
+++ b/analytics/views.py
@@ -1,215 +1,33 @@
+from __future__ import absolute_import
+from __future__ import division
+from six import text_type
+from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union

-from django.conf import settings
-from django.core import urlresolvers
 from django.db import connection
-from django.db.models import Sum
 from django.db.models.query import QuerySet
-from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from django.template import RequestContext, loader
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import ugettext as _
-from django.shortcuts import render
+from django.core import urlresolvers
+from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from jinja2 import Markup as mark_safe

-from analytics.lib.counts import CountStat, process_count_stat, COUNT_STATS
-from analytics.lib.time_utils import time_range
-from analytics.models import BaseCount, InstallationCount, RealmCount, \
-    UserCount, StreamCount, last_successful_fill
-
-from zerver.decorator import has_request_variables, REQ, require_server_admin, \
-    zulip_login_required, to_non_negative_int, to_utc_datetime
-from zerver.lib.request import JsonableError
-from zerver.lib.response import json_success
-from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, \
-    timestamp_to_datetime, convert_to_UTC
-from zerver.models import Realm, UserProfile, UserActivity, \
-    UserActivityInterval, Client
+from zerver.decorator import has_request_variables, REQ, zulip_internal
+from zerver.models import get_realm, UserActivity, UserActivityInterval, Realm
+from zerver.lib.timestamp import timestamp_to_datetime

 from collections import defaultdict
 from datetime import datetime, timedelta
 import itertools
-import json
-import logging
-import pytz
-import re
 import time
-
-from six.moves import filter, map, range, zip
-from typing import Any, Callable, Dict, List, Optional, Set, Text, \
-    Tuple, Type, Union
-
-@zulip_login_required
-def stats(request):
-    # type: (HttpRequest) -> HttpResponse
-    return render(request,
-                  'analytics/stats.html',
-                  context=dict(realm_name = request.user.realm.name))
-
-@has_request_variables
-def get_chart_data(request, user_profile, chart_name=REQ(),
-                   min_length=REQ(converter=to_non_negative_int, default=None),
-                   start=REQ(converter=to_utc_datetime, default=None),
-                   end=REQ(converter=to_utc_datetime, default=None)):
-    # type: (HttpRequest, UserProfile, Text, Optional[int], Optional[datetime], Optional[datetime]) -> HttpResponse
-    if chart_name == 'number_of_humans':
-        stat = COUNT_STATS['realm_active_humans::day']
-        tables = [RealmCount]
-        subgroup_to_label = {None: 'human'}  # type: Dict[Optional[str], str]
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_over_time':
-        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        tables = [RealmCount, UserCount]
-        subgroup_to_label = {'false': 'human', 'true': 'bot'}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_by_message_type':
-        stat = COUNT_STATS['messages_sent:message_type:day']
-        tables = [RealmCount, UserCount]
-        subgroup_to_label = {'public_stream': 'Public streams',
-                             'private_stream': 'Private streams',
-                             'private_message': 'Private messages',
-                             'huddle_message': 'Group private messages'}
-        labels_sort_function = lambda data: sort_by_totals(data['realm'])
-        include_empty_subgroups = True
-    elif chart_name == 'messages_sent_by_client':
-        stat = COUNT_STATS['messages_sent:client:day']
-        tables = [RealmCount, UserCount]
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {str(id): name for id, name in Client.objects.values_list('id', 'name')}
-        labels_sort_function = sort_client_labels
-        include_empty_subgroups = False
-    else:
-        raise JsonableError(_("Unknown chart name: %s") % (chart_name,))
-
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None:
-        start = convert_to_UTC(start)
-    if end is not None:
-        end = convert_to_UTC(end)
-    if start is not None and end is not None and start > end:
-        raise JsonableError(_("Start time is later than end time. Start: %(start)s, End: %(end)s") %
-                            {'start': start, 'end': end})
-
-    realm = user_profile.realm
-    if start is None:
-        start = realm.date_created
-    if end is None:
-        end = last_successful_fill(stat.property)
-    if end is None or start > end:
-        logging.warning("User from realm %s attempted to access /stats, but the computed "
-                        "start time: %s (creation time of realm) is later than the computed "
-                        "end time: %s (last successful analytics update). Is the "
-                        "analytics cron job running?" % (realm.string_id, start, end))
-        raise JsonableError(_("No analytics data available. Please contact your server administrator."))
-
-    end_times = time_range(start, end, stat.frequency, min_length)
-    data = {'end_times': end_times, 'frequency': stat.frequency}
-    for table in tables:
-        if table == RealmCount:
-            data['realm'] = get_time_series_by_subgroup(
-                stat, RealmCount, realm.id, end_times, subgroup_to_label, include_empty_subgroups)
-        if table == UserCount:
-            data['user'] = get_time_series_by_subgroup(
-                stat, UserCount, user_profile.id, end_times, subgroup_to_label, include_empty_subgroups)
-    if labels_sort_function is not None:
-        data['display_order'] = labels_sort_function(data)
-    else:
-        data['display_order'] = None
-    return json_success(data=data)
-
-def sort_by_totals(value_arrays):
-    # type: (Dict[str, List[int]]) -> List[str]
-    totals = [(sum(values), label) for label, values in value_arrays.items()]
-    totals.sort(reverse=True)
-    return [label for total, label in totals]
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data):
-    # type: (Dict[str, Dict[str, List[int]]]) -> List[str]
-    realm_order = sort_by_totals(data['realm'])
-    user_order = sort_by_totals(data['user'])
-    label_sort_values = {}  # type: Dict[str, float]
-    for i, label in enumerate(realm_order):
-        label_sort_values[label] = i
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i-.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(),
-                                                  key=lambda x: x[1])]
-
-def table_filtered_to_id(table, key_id):
-    # type: (Type[BaseCount], int) -> QuerySet
-    if table == RealmCount:
-        return RealmCount.objects.filter(realm_id=key_id)
-    elif table == UserCount:
-        return UserCount.objects.filter(user_id=key_id)
-    elif table == StreamCount:
-        return StreamCount.objects.filter(stream_id=key_id)
-    elif table == InstallationCount:
-        return InstallationCount.objects.all()
-    else:
-        raise AssertionError("Unknown table: %s" % (table,))
-
-def client_label_map(name):
-    # type: (str) -> str
-    if name == "website":
-        return "Website"
-    if name.startswith("desktop app"):
-        return "Old desktop app"
-    if name == "ZulipElectron":
-        return "Desktop app"
-    if name == "ZulipAndroid":
-        return "Old Android app"
-    if name == "ZulipiOS":
-        return "Old iOS app"
-    if name == "ZulipMobile":
-        return "Mobile app"
-    if name in ["ZulipPython", "API: Python"]:
-        return "Python API"
-    if name.startswith("Zulip") and name.endswith("Webhook"):
-        return name[len("Zulip"):-len("Webhook")] + " webhook"
-    return name
-
-def rewrite_client_arrays(value_arrays):
-    # type: (Dict[str, List[int]]) -> Dict[str, List[int]]
-    mapped_arrays = {}  # type: Dict[str, List[int]]
-    for label, array in value_arrays.items():
-        mapped_label = client_label_map(label)
-        if mapped_label in mapped_arrays:
-            for i in range(0, len(array)):
-                mapped_arrays[mapped_label][i] += value_arrays[label][i]
-        else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
-    return mapped_arrays
-
-def get_time_series_by_subgroup(stat, table, key_id, end_times, subgroup_to_label, include_empty_subgroups):
-    # type: (CountStat, Type[BaseCount], int, List[datetime], Dict[Optional[str], str], bool) -> Dict[str, List[int]]
-    queryset = table_filtered_to_id(table, key_id).filter(property=stat.property) \
-                                                  .values_list('subgroup', 'end_time', 'value')
-    value_dicts = defaultdict(lambda: defaultdict(int))  # type: Dict[Optional[str], Dict[datetime, int]]
-    for subgroup, end_time, value in queryset:
-        value_dicts[subgroup][end_time] = value
-    value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
-        if (subgroup in value_dicts) or include_empty_subgroups:
-            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
-
-    if stat == COUNT_STATS['messages_sent:client:day']:
-        # HACK: We rewrite these arrays to collapse the Client objects
-        # with similar names into a single sum, and generally give
-        # them better names
-        return rewrite_client_arrays(value_arrays)
-    return value_arrays
-
-
+import re
+import pytz
+from six.moves import filter
+from six.moves import map
+from six.moves import range
+from six.moves import zip
 eastern_tz = pytz.timezone('US/Eastern')

+from zproject.jinja2 import render_to_response
+
 def make_table(title, cols, rows, has_row_class=False):
    # type: (str, List[str], List[Any], bool) -> str

@@ -242,7 +60,7 @@ def get_realm_day_counts():
    # type: () -> Dict[str, Dict[str, str]]
    query = '''
        select
-            r.string_id,
+            r.domain,
            (now()::date - pub_date::date) age,
            count(*) cnt
        from zerver_message m
@@ -256,10 +74,10 @@ def get_realm_day_counts():
        and
            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
        group by
-            r.string_id,
+            r.domain,
            age
        order by
-            r.string_id,
+            r.domain,
            age
    '''
    cursor = connection.cursor()
@@ -267,13 +85,14 @@ def get_realm_day_counts():
    rows = dictfetchall(cursor)
    cursor.close()

-    counts = defaultdict(dict)  # type: Dict[str, Dict[int, int]]
+    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
    for row in rows:
-        counts[row['string_id']][row['age']] = row['cnt']
+        counts[row['domain']][row['age']] = row['cnt']
+

    result = {}
-    for string_id in counts:
-        raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
+    for domain in counts:
+        raw_cnts = [counts[domain].get(age, 0) for age in range(8)]
        min_cnt = min(raw_cnts)
        max_cnt = max(raw_cnts)

@@ -289,7 +108,7 @@ def get_realm_day_counts():
            return '<td class="number %s">%s</td>' % (good_bad, cnt)

        cnts = ''.join(map(format_count, raw_cnts))
-        result[string_id] = dict(cnts=cnts)
+        result[domain] = dict(cnts=cnts)

    return result

@@ -297,7 +116,7 @@ def realm_summary_table(realm_minutes):
    # type: (Dict[str, float]) -> str
    query = '''
        SELECT
-            realm.string_id,
+            realm.domain,
            coalesce(user_counts.active_user_count, 0) active_user_count,
            coalesce(at_risk_counts.at_risk_count, 0) at_risk_count,
            (
@@ -390,7 +209,7 @@ def realm_summary_table(realm_minutes):
                AND
                    last_visit > now() - interval '2 week'
        )
-        ORDER BY active_user_count DESC, string_id ASC
+        ORDER BY active_user_count DESC, domain ASC
        '''

    cursor = connection.cursor()
@@ -402,26 +221,26 @@ def realm_summary_table(realm_minutes):
    counts = get_realm_day_counts()
    for row in rows:
        try:
-            row['history'] = counts[row['string_id']]['cnts']
-        except Exception:
+            row['history'] = counts[row['domain']]['cnts']
+        except:
            row['history'] = ''

    # augment data with realm_minutes
    total_hours = 0.0
    for row in rows:
-        string_id = row['string_id']
-        minutes = realm_minutes.get(string_id, 0.0)
+        domain = row['domain']
+        minutes = realm_minutes.get(domain, 0.0)
        hours = minutes / 60.0
        total_hours += hours
        row['hours'] = str(int(hours))
        try:
            row['hours_per_user'] = '%.1f' % (hours / row['active_user_count'],)
-        except Exception:
+        except:
            pass

    # formatting
    for row in rows:
-        row['string_id'] = realm_activity_link(row['string_id'])
+        row['domain'] = realm_activity_link(row['domain'])

    # Count active sites
    def meets_goal(row):
@@ -441,8 +260,9 @@ def realm_summary_table(realm_minutes):
        total_bot_count += int(row['bot_count'])
        total_at_risk_count += int(row['at_risk_count'])

+
    rows.append(dict(
-        string_id='Total',
+        domain='Total',
        active_user_count=total_active_user_count,
        user_profile_count=total_user_profile_count,
        bot_count=total_bot_count,
@@ -475,20 +295,20 @@ def user_activity_intervals():
        'start',
        'end',
        'user_profile__email',
-        'user_profile__realm__string_id'
+        'user_profile__realm__domain'
    ).order_by(
-        'user_profile__realm__string_id',
+        'user_profile__realm__domain',
        'user_profile__email'
    )

-    by_string_id = lambda row: row.user_profile.realm.string_id
+    by_domain = lambda row: row.user_profile.realm.domain
    by_email = lambda row: row.user_profile.email

    realm_minutes = {}

-    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
+    for domain, realm_intervals in itertools.groupby(all_intervals, by_domain):
        realm_duration = timedelta(0)
-        output += '<hr>%s\n' % (string_id,)
+        output += '<hr>%s\n' % (domain,)
        for email, intervals in itertools.groupby(realm_intervals, by_email):
            duration = timedelta(0)
            for interval in intervals:
@@ -500,7 +320,7 @@ def user_activity_intervals():
            realm_duration += duration
            output += "  %-*s%s\n" % (37, email, duration)

-        realm_minutes[string_id] = realm_duration.total_seconds() / 60
+        realm_minutes[domain] = realm_duration.total_seconds() / 60

    output += "\nTotal Duration:                      %s\n" % (total_duration,)
    output += "\nTotal Duration in minutes:           %s\n" % (total_duration.total_seconds() / 60.,)
@@ -538,7 +358,7 @@ def sent_messages_report(realm):
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
-                r.string_id = %s
+                r.domain = %s
            and
                (not up.is_bot)
            and
@@ -557,7 +377,7 @@ def sent_messages_report(realm):
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
-                r.string_id = %s
+                r.domain = %s
            and
                up.is_bot
            and
@@ -592,7 +412,7 @@ def ad_hoc_queries():
                row[i] = fixup_func(row[i])

        for i, col in enumerate(cols):
-            if col == 'Realm':
+            if col == 'Domain':
                fix_rows(i, realm_activity_link)
            elif col in ['Last time', 'Last visit']:
                fix_rows(i, format_date_for_activity_reports)
@@ -613,7 +433,7 @@ def ad_hoc_queries():

        query = '''
            select
-                realm.string_id,
+                realm.domain,
                up.id user_id,
                client.name,
                sum(count) as hits,
@@ -624,13 +444,13 @@ def ad_hoc_queries():
            join zerver_realm realm on realm.id = up.realm_id
            where
                client.name like '%s'
-            group by string_id, up.id, client.name
+            group by domain, up.id, client.name
            having max(last_visit) > now() - interval '2 week'
-            order by string_id, up.id, client.name
+            order by domain, up.id, client.name
        ''' % (mobile_type,)

        cols = [
-            'Realm',
+            'Domain',
            'User id',
            'Name',
            'Hits',
@@ -645,7 +465,7 @@ def ad_hoc_queries():

    query = '''
        select
-            realm.string_id,
+            realm.domain,
            client.name,
            sum(count) as hits,
            max(last_visit) as last_time
@@ -655,13 +475,13 @@ def ad_hoc_queries():
        join zerver_realm realm on realm.id = up.realm_id
        where
            client.name like 'desktop%%'
-        group by string_id, client.name
+        group by domain, client.name
        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client.name
+        order by domain, client.name
    '''

    cols = [
-        'Realm',
+        'Domain',
        'Client',
        'Hits',
        'Last time'
@@ -671,11 +491,11 @@ def ad_hoc_queries():

    ###

-    title = 'Integrations by realm'
+    title = 'Integrations by domain'

    query = '''
        select
-            realm.string_id,
+            realm.domain,
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
@@ -693,13 +513,13 @@ def ad_hoc_queries():
            )
        or
            query like '%%external%%'
-        group by string_id, client_name
+        group by domain, client_name
        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client_name
+        order by domain, client_name
    '''

    cols = [
-        'Realm',
+        'Domain',
        'Client',
        'Hits',
        'Last time'
@@ -717,7 +537,7 @@ def ad_hoc_queries():
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
-            realm.string_id,
+            realm.domain,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
@@ -731,14 +551,14 @@ def ad_hoc_queries():
            )
        or
            query like '%%external%%'
-        group by client_name, string_id
+        group by client_name, domain
        having max(last_visit) > now() - interval '2 week'
-        order by client_name, string_id
+        order by client_name, domain
    '''

    cols = [
        'Client',
-        'Realm',
+        'Domain',
        'Hits',
        'Last time'
    ]
@@ -747,12 +567,12 @@ def ad_hoc_queries():

    return pages

-@require_server_admin
+@zulip_internal
@has_request_variables
 def get_activity(request):
    # type: (HttpRequest) -> HttpResponse
-    duration_content, realm_minutes = user_activity_intervals()  # type: Tuple[mark_safe, Dict[str, float]]
-    counts_content = realm_summary_table(realm_minutes)  # type: str
+    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
+    counts_content = realm_summary_table(realm_minutes) # type: str
    data = [
        ('Counts', counts_content),
        ('Durations', duration_content),
@@ -762,10 +582,10 @@ def get_activity(request):

    title = 'Activity'

-    return render(
-        request,
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, title=title, is_home=True),
+        dict(data=data, title=title, is_home=True),
+        request=request
    )

 def get_user_activity_records_for_realm(realm, is_bot):
@@ -780,9 +600,9 @@ def get_user_activity_records_for_realm(realm, is_bot):
    ]

    records = UserActivity.objects.filter(
-        user_profile__realm__string_id=realm,
-        user_profile__is_active=True,
-        user_profile__is_bot=is_bot
+            user_profile__realm__domain=realm,
+            user_profile__is_active=True,
+            user_profile__is_bot=is_bot
    )
    records = records.order_by("user_profile__email", "-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
@@ -799,7 +619,7 @@ def get_user_activity_records_for_email(email):
    ]

    records = UserActivity.objects.filter(
-        user_profile__email=email
+            user_profile__email=email
    )
    records = records.order_by("-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
@@ -817,10 +637,10 @@ def raw_user_activity_table(records):
    def row(record):
        # type: (QuerySet) -> List[Any]
        return [
-            record.query,
-            record.client.name,
-            record.count,
-            format_date_for_activity_reports(record.last_visit)
+                record.query,
+                record.client.name,
+                record.count,
+                format_date_for_activity_reports(record.last_visit)
        ]

    rows = list(map(row, records))
@@ -834,20 +654,19 @@ def get_user_activity_summary(records):
    #: We could use something like:
    # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
    #: but that would require this long `Union` to carry on throughout inner functions.
-    summary = {}  # type: Dict[str, Dict[str, Any]]
-
+    summary = {} # type: Dict[str, Dict[str, Any]]
    def update(action, record):
        # type: (str, QuerySet) -> None
        if action not in summary:
            summary[action] = dict(
-                count=record.count,
-                last_visit=record.last_visit
+                    count=record.count,
+                    last_visit=record.last_visit
            )
        else:
            summary[action]['count'] += record.count
            summary[action]['last_visit'] = max(
-                summary[action]['last_visit'],
-                record.last_visit
+                    summary[action]['last_visit'],
+                    record.last_visit
            )

    if records:
@@ -875,6 +694,7 @@ def get_user_activity_summary(records):
            update('pointer', record)
        update(client, record)

+
    return summary

 def format_date_for_activity_reports(date):
@@ -891,23 +711,23 @@ def user_activity_link(email):
    email_link = '<a href="%s">%s</a>' % (url, email)
    return mark_safe(email_link)

-def realm_activity_link(realm_str):
+def realm_activity_link(realm):
    # type: (str) -> mark_safe
    url_name = 'analytics.views.get_realm_activity'
-    url = urlresolvers.reverse(url_name, kwargs=dict(realm_str=realm_str))
-    realm_link = '<a href="%s">%s</a>' % (url, realm_str)
+    url = urlresolvers.reverse(url_name, kwargs=dict(realm=realm))
+    realm_link = '<a href="%s">%s</a>' % (url, realm)
    return mark_safe(realm_link)

 def realm_client_table(user_summaries):
    # type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
    exclude_keys = [
-        'internal',
-        'name',
-        'use',
-        'send',
-        'pointer',
-        'website',
-        'desktop',
+            'internal',
+            'name',
+            'use',
+            'send',
+            'pointer',
+            'website',
+            'desktop',
    ]

    rows = []
@@ -921,22 +741,22 @@ def realm_client_table(user_summaries):
            count = v['count']
            last_visit = v['last_visit']
            row = [
-                format_date_for_activity_reports(last_visit),
-                client,
-                name,
-                email_link,
-                count,
+                    format_date_for_activity_reports(last_visit),
+                    client,
+                    name,
+                    email_link,
+                    count,
            ]
            rows.append(row)

    rows = sorted(rows, key=lambda r: r[0], reverse=True)

    cols = [
-        'Last visit',
-        'Client',
-        'Name',
-        'Email',
-        'Count',
+            'Last visit',
+            'Client',
+            'Name',
+            'Email',
+            'Count',
    ]

    title = 'Clients'
@@ -953,25 +773,25 @@ def user_activity_summary_table(user_summary):
        count = v['count']
        last_visit = v['last_visit']
        row = [
-            format_date_for_activity_reports(last_visit),
-            client,
-            count,
+                format_date_for_activity_reports(last_visit),
+                client,
+                count,
        ]
        rows.append(row)

    rows = sorted(rows, key=lambda r: r[0], reverse=True)

    cols = [
-        'last_visit',
-        'client',
-        'count',
+            'last_visit',
+            'client',
+            'count',
    ]

    title = 'User Activity'
    return make_table(title, cols, rows)

 def realm_user_summary_table(all_records, admin_emails):
-    # type: (List[QuerySet], Set[Text]) -> Tuple[Dict[str, Dict[str, Any]], str]
+    # type: (List[QuerySet], Set[text_type]) -> Tuple[Dict[str, Dict[str, Any]], str]
    user_records = {}

    def by_email(record):
@@ -997,7 +817,7 @@ def realm_user_summary_table(all_records, admin_emails):

    def is_recent(val):
        # type: (Optional[datetime]) -> bool
-        age = timezone_now() - val
+        age = datetime.now(val.tzinfo) - val # type: ignore # datetie.now tzinfo bug.
        return age.total_seconds() < 5 * 60

    rows = []
@@ -1019,21 +839,21 @@ def realm_user_summary_table(all_records, admin_emails):
        rows.append(row)

    def by_used_time(row):
-        # type: (Dict[str, Any]) -> str
+        # type: (Dict[str, Sequence[str]]) -> str
        return row['cells'][3]

    rows = sorted(rows, key=by_used_time, reverse=True)

    cols = [
-        'Name',
-        'Email',
-        'Total sent',
-        'Heard from',
-        'Message sent',
-        'Pointer motion',
-        'Desktop',
-        'ZulipiOS',
-        'Android',
+            'Name',
+            'Email',
+            'Total sent',
+            'Heard from',
+            'Message sent',
+            'Pointer motion',
+            'Desktop',
+            'ZulipiOS',
+            'Android'
    ]

    title = 'Summary'
@@ -1041,21 +861,21 @@ def realm_user_summary_table(all_records, admin_emails):
    content = make_table(title, cols, rows, has_row_class=True)
    return user_records, content

-@require_server_admin
-def get_realm_activity(request, realm_str):
+@zulip_internal
+def get_realm_activity(request, realm):
    # type: (HttpRequest, str) -> HttpResponse
-    data = []  # type: List[Tuple[str, str]]
-    all_user_records = {}  # type: Dict[str, Any]
+    data = [] # type: List[Tuple[str, str]]
+    all_user_records = {} # type: Dict[str, Any]

    try:
-        admins = Realm.objects.get(string_id=realm_str).get_admin_users()
+        admins = get_realm(realm).get_admin_users()
    except Realm.DoesNotExist:
-        return HttpResponseNotFound("Realm %s does not exist" % (realm_str,))
+        return HttpResponseNotFound("Realm %s does not exist" % (realm,))

    admin_emails = {admin.email for admin in admins}

-    for is_bot, page_title in [(False, 'Humans'), (True, 'Bots')]:
-        all_records = list(get_user_activity_records_for_realm(realm_str, is_bot))
+    for is_bot, page_title in [(False,  'Humans'), (True, 'Bots')]:
+        all_records = list(get_user_activity_records_for_realm(realm, is_bot))

        user_records, content = realm_user_summary_table(all_records, admin_emails)
        all_user_records.update(user_records)
@@ -1066,23 +886,29 @@ def get_realm_activity(request, realm_str):
    content = realm_client_table(all_user_records)
    data += [(page_title, content)]

+
    page_title = 'History'
-    content = sent_messages_report(realm_str)
+    content = sent_messages_report(realm)
    data += [(page_title, content)]

-    title = realm_str
-    return render(
-        request,
+    fix_name = lambda realm: realm.replace('.', '_')
+
+    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
+    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (fix_name(realm),)
+
+    title = realm
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, realm_link=None, title=title),
+        dict(data=data, realm_link=realm_link, title=title),
+        request=request
    )

-@require_server_admin
+@zulip_internal
 def get_user_activity(request, email):
    # type: (HttpRequest, str) -> HttpResponse
    records = get_user_activity_records_for_email(email)

-    data = []  # type: List[Tuple[str, str]]
+    data = [] # type: List[Tuple[str, str]]
    user_summary = get_user_activity_summary(records)
    content = user_activity_summary_table(user_summary)

@@ -1092,8 +918,8 @@ def get_user_activity(request, email):
    data += [('Info', content)]

    title = email
-    return render(
-        request,
+    return render_to_response(
        'analytics/activity.html',
-        context=dict(data=data, title=title),
+        dict(data=data, title=title),
+        request=request
    )
--- a/api/MANIFEST.in
+++ b/api/MANIFEST.in
@@ -0,0 +1,11 @@
+recursive-include integrations *
+include README.md
+include examples/zuliprc
+include examples/send-message
+include examples/subscribe
+include examples/get-public-streams
+include examples/unsubscribe
+include examples/list-members
+include examples/list-subscriptions
+include examples/print-messages
+include examples/recent-messages
--- a/api/README.md
+++ b/api/README.md
@@ -0,0 +1,159 @@
+#### Dependencies
+
+The [Zulip API](https://zulip.com/api) Python bindings require the
+following Python libraries:
+
+* simplejson
+* requests (version >= 0.12.1)
+
+
+#### Installing
+
+This package uses distutils, so you can just run:
+
+    python setup.py install
+
+#### Using the API
+
+For now, the only fully supported API operation is sending a message.
+The other API queries work, but are under active development, so
+please make sure we know you're using them so that we can notify you
+as we make any changes to them.
+
+The easiest way to use these API bindings is to base your tools off
+of the example tools under examples/ in this distribution.
+
+If you place your API key in the config file `~/.zuliprc` the Python
+API bindings will automatically read it in. The format of the config
+file is as follows:
+
+    [api]
+    key=<api key from the web interface>
+    email=<your email address>
+    site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    site=https://api.zulip.com
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
+
+Alternatively, you may explicitly use "--user" and "--api-key" in our
+examples, which is especially useful if you are running several bots
+which share a home directory.
+
+The command line equivalents for other configuration options are:
+
+    --site=<your Zulip server's URI>
+    --insecure
+    --cert-bundle=<file>
+
+You can obtain your Zulip API key, create bots, and manage bots all
+from your Zulip [settings page](https://zulip.com/#settings).
+
+A typical simple bot sending API messages will look as follows:
+
+At the top of the file:
+
+    # Make sure the Zulip API distribution's root directory is in sys.path, then:
+    import zulip
+    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
+
+When you want to send a message:
+
+    message = {
+      "type": "stream",
+      "to": ["support"],
+      "subject": "your subject",
+      "content": "your content",
+    }
+    zulip_client.send_message(message)
+
+Additional examples:
+
+    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
+                         'subject': 'feedback', 'to': ['support']})
+    client.send_message({'type': 'private', 'content': 'Zulip rules!',
+                         'to': ['user1@example.com', 'user2@example.com']})
+
+send_message() returns a dict guaranteed to contain the following
+keys: msg, result.  For successful calls, result will be "success" and
+msg will be the empty string.  On error, result will be "error" and
+msg will describe what went wrong.
+
+#### Logging
+The Zulip API comes with a ZulipStream class which can be used with the
+logging module:
+
+```
+import zulip
+import logging
+stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
+logger = logging.getLogger("your_logger")
+logger.addHandler(logging.StreamHandler(stream))
+logger.setLevel(logging.DEBUG)
+logger.info("This is an INFO test.")
+logger.debug("This is a DEBUG test.")
+logger.warn("This is a WARN test.")
+logger.error("This is a ERROR test.")
+```
+
+#### Sending messages
+
+You can use the included `zulip-send` script to send messages via the
+API directly from existing scripts.
+
+    zulip-send hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+Alternatively, if you don't want to use your ~/.zuliprc file:
+
+    zulip-send --user shakespeare-bot@example.com \
+        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+        hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.
--- a/api/bin/zulip-send
+++ b/api/bin/zulip-send
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# zulip-send -- Sends a message to the specified recipients.
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import os
+import optparse
+import logging
+
+
+logging.basicConfig()
+
+log = logging.getLogger('zulip-send')
+
+def do_send_message(client, message_data ):
+    '''Sends a message and optionally prints status about the same.'''
+
+    if message_data['type'] == 'stream':
+        log.info('Sending message to stream "%s", subject "%s"... ' % \
+            (message_data['to'], message_data['subject']))
+    else:
+        log.info('Sending message to %s... ' % message_data['to'])
+    response = client.send_message(message_data)
+    if response['result'] == 'success':
+        log.info('Message sent.')
+        return True
+    else:
+        log.error(response['msg'])
+        return False
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    usage = """%prog [options] [recipient...]
+
+    Sends a message specified recipients.
+
+    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
+              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
+
+    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
+    '--user' and '--api-key' arguments.
+    """
+
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+    import zulip
+
+    parser = optparse.OptionParser(usage=usage)
+
+    # Grab parser options from the API common set
+    parser.add_option_group(zulip.generate_option_group(parser))
+
+    parser.add_option('-m', '--message',
+                      help='Specifies the message to send, prevents interactive prompting.')
+
+    group = optparse.OptionGroup(parser, 'Stream parameters')
+    group.add_option('-s', '--stream',
+                      dest='stream',
+                      action='store',
+                      help='Allows the user to specify a stream for the message.')
+    group.add_option('-S', '--subject',
+                      dest='subject',
+                      action='store',
+                      help='Allows the user to specify a subject for the message.')
+    parser.add_option_group(group)
+
+
+    (options, recipients) = parser.parse_args(argv[1:])
+
+    if options.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+    # Sanity check user data
+    if len(recipients) != 0 and (options.stream or options.subject):
+        parser.error('You cannot specify both a username and a stream/subject.')
+    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
+        parser.error('Stream messages must have a subject')
+    if len(recipients) == 0 and not (options.stream and options.subject):
+        parser.error('You must specify a stream/subject or at least one recipient.')
+
+    client = zulip.init_from_options(options)
+
+    if not options.message:
+        options.message = sys.stdin.read()
+
+    if options.stream:
+        message_data = {
+            'type': 'stream',
+            'content': options.message,
+            'subject': options.subject,
+            'to': options.stream,
+        }
+    else:
+        message_data = {
+            'type': 'private',
+            'content': options.message,
+            'to': recipients,
+        }
+
+    if not do_send_message(client, message_data):
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())
--- a/api/examples/create-user
+++ b/api/examples/create-user
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+from os import path
+import optparse
+
+usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
+
+Create a user. You must be a realm admin to use this API, and the user
+will be created in your realm.
+
+Example: create-user --site=http://localhost:9991 --user=rwbarton@zulip.com --new-email=jarthur@zulip.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
+"""
+
+sys.path.append(path.join(path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--new-email')
+parser.add_option('--new-password')
+parser.add_option('--new-full-name')
+parser.add_option('--new-short-name')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.create_user({
+        'email': options.new_email,
+        'password': options.new_password,
+        'full_name': options.new_full_name,
+        'short_name': options.new_short_name
+        }))
--- a/api/examples/edit-message
+++ b/api/examples/edit-message
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
+
+Edits a message that you sent
+
+Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--message-id', default="")
+parser.add_option('--subject', default="")
+parser.add_option('--content',   default="")
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "message_id": options.message_id,
+}
+if options.subject != "":
+    message_data["subject"] = options.subject
+if options.content != "":
+    message_data["content"] = options.content
+print(client.update_message(message_data))
--- a/api/examples/get-public-streams
+++ b/api/examples/get-public-streams
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out all the public streams in the realm.
+
+Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_streams(include_public=True, include_subscribed=False))
--- a/api/examples/list-members
+++ b/api/examples/list-members
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
+
+List the names and e-mail addresses of the people in your realm.
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+for user in client.get_members()["members"]:
+    print(user["full_name"], user["email"])
--- a/api/examples/list-subscriptions
+++ b/api/examples/list-subscriptions
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out a list of the user's subscriptions.
+
+Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.list_subscriptions())
--- a/api/examples/print-events
+++ b/api/examples/print-events
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out certain events received by the indicated bot or user matching the filter below.
+
+Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_event(event):
+    print(event)
+
+# This is a blocking call, and will continuously poll for new events
+# Note also the filter here is messages to the stream Denmark; if you
+# don't specify event_types it'll print all events.
+client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])
--- a/api/examples/print-messages
+++ b/api/examples/print-messages
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out each message received by the indicated bot or user.
+
+Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_message(message):
+    print(message)
+
+# This is a blocking call, and will continuously poll for new messages
+client.call_on_each_message(print_message)
--- a/api/examples/print-next-message
+++ b/api/examples/print-next-message
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-next-message --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out the next message received by the user.
+
+Example: print-next-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_messages({}))
--- a/api/examples/recent-messages
+++ b/api/examples/recent-messages
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages recieved by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+req = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.do_api_query(req, zulip.API_VERSTRING + 'messages', method='GET')
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])
--- a/api/examples/send-message
+++ b/api/examples/send-message
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
+
+Sends a test message to the specified recipients.
+
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--subject', default="test")
+parser.add_option('--message', default="test message")
+parser.add_option('--type',   default='private')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if len(args) == 0:
+    parser.error("You must specify recipients")
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "type": options.type,
+    "content": options.message,
+    "subject": options.subject,
+    "to": args,
+}
+print(client.send_message(message_data))
--- a/api/examples/subscribe
+++ b/api/examples/subscribe
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is subscribed to the listed streams.
+
+Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))
--- a/api/examples/unsubscribe
+++ b/api/examples/unsubscribe
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is not subscribed to the listed streams.
+
+Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.remove_subscriptions(options.streams.split()))
--- a/api/examples/zuliprc
+++ b/api/examples/zuliprc
@@ -0,0 +1,4 @@
+; Save this file as ~/.zuliprc
+[api]
+key=<your bot's api key from the web interface>
+email=<your bot's email address>
--- a/api/integrations/asana/zulip_asana_config.py
+++ b/api/integrations/asana/zulip_asana_config.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+### REQUIRED CONFIGURATION ###
+
+# Change these values to your Asana credentials.
+ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# Change these values to the credentials for your Asana bot.
+ZULIP_USER = "asana-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The Zulip stream that will receive Asana task updates.
+ZULIP_STREAM_NAME = "asana"
+
+
+### OPTIONAL CONFIGURATION ###
+
+# Set to None for logging to stdout when testing, and to a file for
+# logging in production.
+#LOG_FILE = "/var/tmp/zulip_asana.log"
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_asana.state"
+
+# When initially started, how many hours of messages to include.
+ASANA_INITIAL_HISTORY_HOURS = 1
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://api.zulip.com"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None
--- a/api/integrations/asana/zulip_asana_mirror
+++ b/api/integrations/asana/zulip_asana_mirror
@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Asana integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_asana_mirror" script is run continuously, possibly on a work computer
+# or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+import base64
+from datetime import datetime, timedelta
+
+import json
+import logging
+import os
+import time
+from six.moves import urllib
+
+import sys
+
+try:
+    import dateutil.parser
+    import dateutil.tz
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_asana_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+if config.LOG_FILE:
+    logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+else:
+    logging.basicConfig(level=logging.INFO)
+
+client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
+                      site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
+
+def fetch_from_asana(path):
+    """
+    Request a resource through the Asana API, authenticating using
+    HTTP basic auth.
+    """
+    auth = base64.encodestring('%s:' % (config.ASANA_API_KEY,))
+    headers = {"Authorization": "Basic %s" % auth}
+
+    url = "https://app.asana.com/api/1.0" + path
+    request = urllib.request.Request(url, None, headers)
+    result = urllib.request.urlopen(request)
+
+    return json.load(result)
+
+def send_zulip(topic, content):
+    """
+    Send a message to Zulip using the configured stream and bot credentials.
+    """
+    message = {"type": "stream",
+               "sender": config.ZULIP_USER,
+               "to": config.ZULIP_STREAM_NAME,
+               "subject": topic,
+               "content": content,
+               }
+    return client.send_message(message)
+
+def datestring_to_datetime(datestring):
+    """
+    Given an ISO 8601 datestring, return the corresponding datetime object.
+    """
+    return dateutil.parser.parse(datestring).replace(
+        tzinfo=dateutil.tz.gettz('Z'))
+
+class TaskDict(dict):
+    """
+    A helper class to turn a dictionary with task information into an
+    object where each of the keys is an attribute for easy access.
+    """
+    def __getattr__(self, field):
+        return self.get(field)
+
+def format_topic(task, projects):
+    """
+    Return a string that will be the Zulip message topic for this task.
+    """
+    # Tasks can be associated with multiple projects, but in practice they seem
+    # to mostly be associated with one.
+    project_name = projects[task.projects[0]["id"]]
+    return "%s: %s" % (project_name, task.name)
+
+def format_assignee(task, users):
+    """
+    Return a string describing the task's assignee.
+    """
+    if task.assignee:
+        assignee_name = users[task.assignee["id"]]
+        assignee_info = "**Assigned to**: %s (%s)" % (
+            assignee_name, task.assignee_status)
+    else:
+        assignee_info = "**Status**: Unassigned"
+
+    return assignee_info
+
+def format_due_date(task):
+    """
+    Return a string describing the task's due date.
+    """
+    if task.due_on:
+        due_date_info = "**Due on**: %s" % (task.due_on,)
+    else:
+        due_date_info = "**Due date**: None"
+    return due_date_info
+
+def format_task_creation_event(task, projects, users):
+    """
+    Format the topic and content for a newly-created task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** created:
+
+~~~ quote
+%s
+~~~
+
+%s
+%s
+""" % (task.name, task.notes, assignee_info, due_date_info)
+    return topic, content
+
+def format_task_completion_event(task, projects, users):
+    """
+    Format the topic and content for a completed task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** completed. :white_check_mark:
+
+%s
+%s
+""" % (task.name, assignee_info, due_date_info)
+    return topic, content
+
+def since():
+    """
+    Return a newness threshold for task events to be processed.
+    """
+    # If we have a record of the last event processed and it is recent, use it,
+    # else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
+    def default_since():
+        return datetime.utcnow() - timedelta(
+            hours=config.ASANA_INITIAL_HISTORY_HOURS)
+
+    if os.path.exists(config.RESUME_FILE):
+        try:
+            with open(config.RESUME_FILE, "r") as f:
+                datestring = f.readline().strip()
+                timestamp = float(datestring)
+                max_timestamp_processed = datetime.fromtimestamp(timestamp)
+                logging.info("Reading from resume file: " + datestring)
+        except (ValueError, IOError) as e:
+            logging.warn("Could not open resume file: %s" % (
+                    e.message or e.strerror,))
+            max_timestamp_processed = default_since()
+    else:
+        logging.info("No resume file, processing an initial history.")
+        max_timestamp_processed = default_since()
+
+    # Even if we can read a timestamp from RESUME_FILE, if it is old don't use
+    # it.
+    return max(max_timestamp_processed, default_since())
+
+def process_new_events():
+    """
+    Forward new Asana task events to Zulip.
+    """
+    # In task queries, Asana only exposes IDs for projects and users, so we need
+    # to look up the mappings.
+    projects = dict((elt["id"], elt["name"]) for elt in \
+                        fetch_from_asana("/projects")["data"])
+    users = dict((elt["id"], elt["name"]) for elt in \
+                     fetch_from_asana("/users")["data"])
+
+    cutoff = since()
+    max_timestamp_processed = cutoff
+    time_operations = (("created_at", format_task_creation_event),
+                       ("completed_at", format_task_completion_event))
+    task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
+                   "modified_at", "due_on", "name", "notes", "projects"]
+
+    # First, gather all of the tasks that need processing. We'll
+    # process them in order.
+    new_events = []
+
+    for project_id in projects:
+        project_url = "/projects/%d/tasks?opt_fields=%s" % (
+            project_id, ",".join(task_fields))
+        tasks = fetch_from_asana(project_url)["data"]
+
+        for task in tasks:
+            task = TaskDict(task)
+
+            for time_field, operation in time_operations:
+                if task[time_field]:
+                    operation_time = datestring_to_datetime(task[time_field])
+                    if operation_time > cutoff:
+                        new_events.append((operation_time, time_field, operation, task))
+
+    new_events.sort()
+    now = datetime.utcnow()
+
+    for operation_time, time_field, operation, task in new_events:
+        # Unfortunately, creating an Asana task is not an atomic operation. If
+        # the task was just created, or is missing basic information, it is
+        # probably because the task is still being filled out -- wait until the
+        # next round to process it.
+        if (time_field == "created_at") and \
+                (now - operation_time < timedelta(seconds=30)):
+            # The task was just created, give the user some time to fill out
+            # more information.
+            return
+
+        if (time_field == "created_at") and (not task.name) and \
+                (now - operation_time < timedelta(seconds=60)):
+            # If this new task hasn't had a name for a full 30 seconds, assume
+            # you don't plan on giving it one.
+            return
+
+        topic, content = operation(task, projects, users)
+        logging.info("Sending Zulip for " + topic)
+        result = send_zulip(topic, content)
+
+        # If the Zulip wasn't sent successfully, don't update the
+        # max timestamp processed so the task has another change to
+        # be forwarded. Exit, giving temporary issues time to
+        # resolve.
+        if not result.get("result"):
+            logging.warn("Malformed result, exiting:")
+            logging.warn(result)
+            return
+
+        if result["result"] != "success":
+            logging.warn(result["msg"])
+            return
+
+        if operation_time > max_timestamp_processed:
+            max_timestamp_processed = operation_time
+
+    if max_timestamp_processed > cutoff:
+        max_datestring = max_timestamp_processed.strftime("%s.%f")
+        logging.info("Updating resume file: " + max_datestring)
+        open(config.RESUME_FILE, 'w').write(max_datestring)
+
+while True:
+    try:
+        process_new_events()
+        time.sleep(5)
+    except KeyboardInterrupt:
+        logging.info("Shutting down...")
+        logging.info("Set LOG_FILE to log to a file instead of stdout.")
+        break
--- a/api/integrations/basecamp/zulip_basecamp_config.py
+++ b/api/integrations/basecamp/zulip_basecamp_config.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+
+# Change these values to configure authentication for basecamp account
+BASECAMP_ACCOUNT_ID = "12345678"
+BASECAMP_USERNAME = "foo@example.com"
+BASECAMP_PASSWORD = "p455w0rd"
+
+# This script will mirror this many hours of history on the first run.
+# On subsequent runs this value is ignored.
+BASECAMP_INITIAL_HISTORY_HOURS = 0
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "basecamp-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+ZULIP_STREAM_NAME = "basecamp"
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://api.zulip.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_basecamp.state"
--- a/api/integrations/basecamp/zulip_basecamp_mirror
+++ b/api/integrations/basecamp/zulip_basecamp_mirror
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Basecamp activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "basecamp-mirror.py" script is run continuously, possibly on a work computer
+# or preferably on a server.
+# You may need to install the python-requests library.
+
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import re
+import sys
+import os
+from datetime import datetime, timedelta
+from six.moves.html_parser import HTMLParser
+import six
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_basecamp_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipBasecamp/" + VERSION)
+user_agent = "Basecamp To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+htmlParser = HTMLParser()
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr("Could not open up log for writing:")
+            sys.stderr(e)
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
+        sys.stderr(e)
+
+# builds the message dict for sending a message with the Zulip API
+def build_message(event):
+    if not ('bucket' in event and 'creator' in event and 'html_url' in event):
+        logging.error("Perhaps the Basecamp API changed behavior? "
+                      "This event doesn't have the expected format:\n%s" %(event,))
+        return None
+    # adjust the topic length to be bounded to 60 characters
+    topic = event['bucket']['name']
+    if len(topic) > 60:
+        topic = topic[0:57] + "..."
+    # get the action and target values
+    action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
+    target = htmlParser.unescape(event.get('target', ''))
+    # Some events have "excerpts", which we blockquote
+    excerpt = htmlParser.unescape(event.get('excerpt', ''))
+    if excerpt.strip() == "":
+        message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
+    else:
+        message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
+    # assemble the message data dict
+    message_data = {
+        "type": "stream",
+        "to": config.ZULIP_STREAM_NAME,
+        "subject": topic,
+        "content": message,
+    }
+    return message_data
+
+# the main run loop for this mirror script
+def run_mirror():
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    try:
+        with open(config.RESUME_FILE) as f:
+            since = f.read()
+        since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
+        assert since, "resume file does not meet expected format"
+        since = since.string
+    except (AssertionError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
+        since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
+    try:
+        # we use an exponential backoff approach when we get 429 (Too Many Requests).
+        sleepInterval = 1
+        while True:
+            time.sleep(sleepInterval)
+            response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
+                                    params={'since': since},
+                                    auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
+                                    headers = {"User-Agent": user_agent})
+            if response.status_code == 200:
+                sleepInterval = 1
+                events = json.loads(response.text)
+                if len(events):
+                    logging.info("Got event(s): %s" % (response.text,))
+            if response.status_code >= 500:
+                logging.error(response.status_code)
+                continue
+            if response.status_code == 429:
+                # exponential backoff
+                sleepInterval *= 2
+                logging.error(response.status_code)
+                continue
+            if response.status_code == 400:
+                logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
+                sys.exit(-1)
+            if response.status_code == 401:
+                logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
+                sys.exit(-1)
+            if len(events):
+                since = events[0]['created_at']
+            for event in reversed(events):
+                message_data = build_message(event)
+                if not message_data:
+                    continue
+                zulip_api_result = client.send_message(message_data)
+                if zulip_api_result['result'] == "success":
+                    logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
+                else:
+                    logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
+                # update 'since' each time in case we get KeyboardInterrupted
+                since = event['created_at']
+                # avoid hitting rate-limit
+                time.sleep(0.2)
+
+    except KeyboardInterrupt:
+        logging.info("Shutting down, please hold")
+        open("events.last", 'w').write(since)
+        logging.info("Done!")
+
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    run_mirror()
--- a/api/integrations/codebase/zulip_codebase_config.py
+++ b/api/integrations/codebase/zulip_codebase_config.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+
+# Change these values to configure authentication for your codebase account
+# Note that this is the Codebase API Username, found in the Settings page
+# for your account
+CODEBASE_API_USERNAME = "foo@example.com"
+CODEBASE_API_KEY = "1234561234567abcdef"
+
+# The URL of your codebase setup
+CODEBASE_ROOT_URL = "https://YOUR_COMPANY.codebasehq.com"
+
+# When initially started, how many hours of messages to include.
+# Note that the Codebase API only returns the 20 latest events,
+# if you have more than 20 events that fit within this window,
+# earlier ones may be lost
+CODEBASE_INITIAL_HISTORY_HOURS = 12
+
+# Change these values to configure Zulip authentication for the plugin
+ZULIP_USER = "codebase-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The streams to send commit information and ticket information to
+ZULIP_COMMITS_STREAM_NAME = "codebase"
+ZULIP_TICKETS_STREAM_NAME = "tickets"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://api.zulip.com"
+
+# If you wish to log to a file rather than stdout/stderr,
+# please fill this out your desired path
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_codebase.state"
--- a/api/integrations/codebase/zulip_codebase_mirror
+++ b/api/integrations/codebase/zulip_codebase_mirror
@@ -0,0 +1,329 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip mirror of Codebase HQ activity
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_codebase_mirror" script is run continuously, possibly on a work
+# computer or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+from __future__ import absolute_import
+import requests
+import logging
+import time
+import sys
+import os
+
+from datetime import datetime, timedelta
+import six
+
+
+try:
+    import dateutil.parser
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_codebase_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipCodebase/" + VERSION)
+user_agent = "Codebase To Zulip Mirroring script (zulip-devel@googlegroups.com)"
+
+# find some form of JSON loader/dumper, with a preference order for speed.
+json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
+
+while len(json_implementations):
+    try:
+        json = __import__(json_implementations.pop(0))
+        break
+    except ImportError:
+        continue
+
+def make_api_call(path):
+    response = requests.get("https://api3.codebasehq.com/%s" % (path,),
+                             auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
+                             params={'raw': True},
+                             headers = {"User-Agent": user_agent,
+                                        "Content-Type": "application/json",
+                                        "Accept": "application/json"})
+    if response.status_code == 200:
+        return json.loads(response.text)
+
+    if response.status_code >= 500:
+        logging.error(response.status_code)
+        return None
+    if response.status_code == 403:
+        logging.error("Bad authorization from Codebase. Please check your credentials")
+        sys.exit(-1)
+    else:
+        logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
+        return None
+
+def make_url(path):
+    return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
+
+def handle_event(event):
+    event = event['event']
+    event_type = event['type']
+    actor_name = event['actor_name']
+
+    raw_props = event.get('raw_properties', {})
+
+    project_link = raw_props.get('project_permalink')
+
+    subject = None
+    content = None
+    if event_type == 'repository_creation':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        project_name = raw_props.get('name')
+        project_repo_type = raw_props.get('scm_type')
+
+        url = make_url("projects/%s" % (project_link,))
+        scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
+
+
+        subject = "Repository %s Created" % (project_name,)
+        content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
+    elif event_type == 'push':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        num_commits = raw_props.get('commits_count')
+        branch = raw_props.get('ref_name')
+        project = raw_props.get('project_name')
+        repo_link = raw_props.get('repository_permalink')
+        deleted_ref = raw_props.get('deleted_ref')
+        new_ref = raw_props.get('new_ref')
+
+        subject = "Push to %s on %s" % (branch, project)
+
+        if deleted_ref:
+            content = "%s deleted branch %s from %s" % (actor_name, branch, project)
+        else:
+            if new_ref:
+                branch = "new branch %s" % (branch,)
+            content = "%s pushed %s commit(s) to %s in project %s:\n\n" % \
+                        (actor_name, num_commits, branch, project)
+            for commit in raw_props.get('commits'):
+                ref = commit.get('ref')
+                url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
+                message = commit.get('message')
+                content += "* [%s](%s): %s\n" % (ref, url, message)
+    elif event_type == 'ticketing_ticket':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        assignee = raw_props.get('assignee')
+        priority = raw_props.get('priority')
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+
+        if assignee is None:
+            assignee = "no one"
+        subject = "#%s: %s" % (num, name)
+        content = """%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" % \
+                    (actor_name, num, url, priority, assignee, name)
+    elif event_type == 'ticketing_note':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        num = raw_props.get('number')
+        name = raw_props.get('subject')
+        body = raw_props.get('content')
+        changes = raw_props.get('changes')
+
+
+        url = make_url("projects/%s/tickets/%s" % (project_link, num))
+        subject = "#%s: %s" % (num, name)
+
+        content = ""
+        if body is not None and len(body) > 0:
+            content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
+
+        if 'status_id' in changes:
+            status_change = changes.get('status_id')
+            content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
+    elif event_type == 'ticketing_milestone':
+        stream = config.ZULIP_TICKETS_STREAM_NAME
+
+        name = raw_props.get('name')
+        identifier = raw_props.get('identifier')
+        url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
+
+        subject = name
+        content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
+    elif event_type == 'comment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        comment = raw_props.get('content')
+        commit = raw_props.get('commit_ref')
+
+        # If there's a commit id, it's a comment to a commit
+        if commit:
+            repo_link = raw_props.get('repository_permalink')
+
+            url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
+
+            subject = "%s commented on %s" % (actor_name, commit)
+            content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
+        else:
+            # Otherwise, this is a Discussion item, and handle it
+            subj = raw_props.get("subject")
+            category = raw_props.get("category")
+            comment_content = raw_props.get("content")
+
+            subject = "Discussion: %s" % (subj,)
+
+            if category:
+                format_str = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~"
+                content = format_str % (actor_name, category, comment_content)
+            else:
+                content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
+
+    elif event_type == 'deployment':
+        stream = config.ZULIP_COMMITS_STREAM_NAME
+
+        start_ref = raw_props.get('start_ref')
+        end_ref = raw_props.get('end_ref')
+        environment = raw_props.get('environment')
+        servers = raw_props.get('servers')
+        repo_link = raw_props.get('repository_permalink')
+
+        start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
+        end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
+        between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (
+            project_link, repo_link, start_ref, end_ref))
+
+        subject = "Deployment to %s" % (environment,)
+
+        content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % \
+                    (actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment)
+        if servers is not None:
+            content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
+
+    elif event_type == 'named_tree':
+        # Docs say named_tree type used for new/deleting branches and tags,
+        # but experimental testing showed that they were all sent as 'push' events
+        pass
+    elif event_type == 'wiki_page':
+        logging.warn("Wiki page notifications not yet implemented")
+    elif event_type == 'sprint_creation':
+        logging.warn("Sprint notifications not yet implemented")
+    elif event_type == 'sprint_ended':
+        logging.warn("Sprint notifications not yet implemented")
+    else:
+        logging.info("Unknown event type %s, ignoring!" % (event_type,))
+
+    if subject and content:
+        if len(subject) > 60:
+            subject = subject[:57].rstrip() + '...'
+
+        res = client.send_message({"type": "stream",
+                                   "to": stream,
+                                   "subject": subject,
+                                   "content": content})
+        if res['result'] == 'success':
+            logging.info("Successfully sent Zulip with id: %s" % (res['id']))
+        else:
+            logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
+
+
+# the main run loop for this mirror script
+def run_mirror():
+    # we should have the right (write) permissions on the resume file, as seen
+    # in check_permissions, but it may still be empty or corrupted
+    def default_since():
+        return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
+
+    try:
+        with open(config.RESUME_FILE) as f:
+            timestamp = f.read()
+        if timestamp == '':
+            since = default_since()
+        else:
+            timestamp = int(timestamp, 10)
+            since = datetime.fromtimestamp(timestamp)
+    except (ValueError, IOError) as e:
+        logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
+        since = default_since()
+
+    try:
+        sleepInterval = 1
+        while True:
+            events = make_api_call("activity")[::-1]
+            if events is not None:
+                sleepInterval = 1
+                for event in events:
+                    timestamp = event.get('event', {}).get('timestamp', '')
+                    event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None)
+                    if event_date > since:
+                        handle_event(event)
+                        since = event_date
+            else:
+                # back off a bit
+                if sleepInterval < 22:
+                    sleepInterval += 4
+            time.sleep(sleepInterval)
+
+    except KeyboardInterrupt:
+        open(config.RESUME_FILE, 'w').write(since.strftime("%s"));
+        logging.info("Shutting down Codebase mirror")
+
+# void function that checks the permissions of the files this script needs.
+def check_permissions():
+    # check that the log file can be written
+    if config.LOG_FILE:
+        try:
+            open(config.LOG_FILE, "w")
+        except IOError as e:
+            sys.stderr("Could not open up log for writing:")
+            sys.stderr(e)
+    # check that the resume file can be written (this creates if it doesn't exist)
+    try:
+        open(config.RESUME_FILE, "a+")
+    except IOError as e:
+        sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
+        sys.stderr(e)
+
+if __name__ == "__main__":
+    if not isinstance(config.RESUME_FILE, six.string_types):
+        sys.stderr("RESUME_FILE path not given; refusing to continue")
+    check_permissions()
+    if config.LOG_FILE:
+        logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+    run_mirror()
--- a/api/integrations/git/post-receive
+++ b/api/integrations/git/post-receive
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-receive hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-receive" script is run after receive-pack has accepted a pack
+# and the repository has been updated.  It is passed arguments in through
+# stdin in the form
+#  <oldrev> <newrev> <refname>
+# For example:
+#  aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
+
+from __future__ import absolute_import
+import os
+import sys
+import subprocess
+import os.path
+
+sys.path.insert(0, os.path.dirname(__file__))
+from . import zulip_git_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipGit/" + VERSION)
+
+# check_output is backported from subprocess.py in Python 2.7
+def check_output(*popenargs, **kwargs):
+    if 'stdout' in kwargs:
+        raise ValueError('stdout argument not allowed, it will be overridden.')
+    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    output, unused_err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        raise subprocess.CalledProcessError(retcode, cmd, output=output)
+    return output
+subprocess.check_output = check_output
+
+def git_repository_name():
+    output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
+    if output.strip() == "true":
+        return os.path.basename(os.getcwd())[:-len(".git")]
+    else:
+        return os.path.basename(os.path.dirname(os.getcwd()))
+
+def git_commit_range(oldrev, newrev):
+    log_cmd = ["git", "log", "--reverse",
+               "--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
+    commits = ''
+    for ln in subprocess.check_output(log_cmd).splitlines():
+        author_email, commit_id, subject  = ln.split(None, 2)
+        if hasattr(config, "format_commit_message"):
+            commits += config.format_commit_message(author_email, subject, commit_id)
+        else:
+            commits += '!avatar(%s) %s\n' % (author_email, subject)
+    return commits
+
+def send_bot_message(oldrev, newrev, refname):
+    repo_name   = git_repository_name()
+    branch      = refname.replace('refs/heads/', '')
+    destination = config.commit_notice_destination(repo_name, branch, newrev)
+    if destination is None:
+        # Don't forward the notice anywhere
+        return
+
+    new_head = newrev[:12]
+    old_head = oldrev[:12]
+
+    if (oldrev == '0000000000000000000000000000000000000000' or
+        newrev == '0000000000000000000000000000000000000000'):
+        # New branch pushed or old branch removed
+        added = ''
+        removed = ''
+    else:
+        added   = git_commit_range(oldrev, newrev)
+        removed = git_commit_range(newrev, oldrev)
+
+    if oldrev == '0000000000000000000000000000000000000000':
+        message = '`%s` was pushed to new branch `%s`' % (new_head, branch)
+    elif newrev == '0000000000000000000000000000000000000000':
+        message = 'branch `%s` was removed (was `%s`)' % (branch, old_head)
+    elif removed:
+        message = '`%s` was pushed to `%s`, **REMOVING**:\n\n%s' % (new_head, branch, removed)
+        if added:
+            message += '\n**and adding**:\n\n' + added
+        message += '\n**A HISTORY REWRITE HAS OCCURRED!**'
+        message += '\n@everyone: Please check your local branches to deal with this.'
+    elif added:
+        message = '`%s` was deployed to `%s` with:\n\n%s' % (new_head, branch, added)
+    else:
+        message = '`%s` was pushed to `%s`... but nothing changed?' % (new_head, branch)
+
+    message_data = {
+        "type": "stream",
+        "to": destination["stream"],
+        "subject": destination["subject"],
+        "content": message,
+    }
+    client.send_message(message_data)
+
+for ln in sys.stdin:
+    oldrev, newrev, refname = ln.strip().split()
+    send_bot_message(oldrev, newrev, refname)
--- a/api/integrations/git/zulip_git_config.py
+++ b/api/integrations/git/zulip_git_config.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "git-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * repo   = the name of the git repository
+# * branch = the name of the branch that was pushed to
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit pushed to "master" to
+# * stream "commits"
+# * topic "master"
+# And similarly for branch "test-post-receive" (for use when testing).
+def commit_notice_destination(repo, branch, commit):
+    if branch in ["master", "test-post-receive"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (branch,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+# Modify this function to change how commits are displayed; the most
+# common customization is to include a link to the commit in your
+# graphical repository viewer, e.g.
+#
+# return '!avatar(%s) [%s](https://example.com/commits/%s)\n' % (author, subject, commit_id)
+def format_commit_message(author, subject, commit_id):
+    return '!avatar(%s) %s\n' % (author, subject)
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/hg/zulip-changegroup.py
+++ b/api/integrations/hg/zulip-changegroup.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip hook for Mercurial changeset pushes.
+# Copyright Â© 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+#
+# This hook is called when changesets are pushed to the master repository (ie
+# `hg push`). See https://zulipchat.com/integrations for installation instructions.
+from __future__ import absolute_import
+
+import zulip
+from six.moves import range
+
+VERSION = "0.9"
+
+def format_summary_line(web_url, user, base, tip, branch, node):
+    """
+    Format the first line of the message, which contains summary
+    information about the changeset and links to the changelog if a
+    web URL has been configured:
+
+    Jane Doe <jane@example.com> pushed 1 commit to master (170:e494a5be3393):
+    """
+    revcount = tip - base
+    plural = "s" if revcount > 1 else ""
+
+    if web_url:
+        shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
+        summary_url = "{shortlog}{tip}?revcount={revcount}".format(
+            shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
+        formatted_commit_count = "[{revcount} commit{s}]({url})".format(
+            revcount=revcount, s=plural, url=summary_url)
+    else:
+        formatted_commit_count = "{revcount} commit{s}".format(
+            revcount=revcount, s=plural)
+
+    return u"**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
+        user=user, commits=formatted_commit_count, branch=branch, tip=tip,
+        node=node[:12])
+
+def format_commit_lines(web_url, repo, base, tip):
+    """
+    Format the per-commit information for the message, including the one-line
+    commit summary and a link to the diff if a web URL has been configured:
+    """
+    if web_url:
+        rev_base_url = web_url.rstrip("/") + "/rev/"
+
+    commit_summaries = []
+    for rev in range(base, tip):
+        rev_node = repo.changelog.node(rev)
+        rev_ctx = repo.changectx(rev_node)
+        one_liner = rev_ctx.description().split("\n")[0]
+
+        if web_url:
+            summary_url = rev_base_url + str(rev_ctx)
+            summary = "* [{summary}]({url})".format(
+                summary=one_liner, url=summary_url)
+        else:
+            summary = "* {summary}".format(summary=one_liner)
+
+        commit_summaries.append(summary)
+
+    return "\n".join(summary for summary in commit_summaries)
+
+def send_zulip(email, api_key, site, stream, subject, content):
+    """
+    Send a message to Zulip using the provided credentials, which should be for
+    a bot in most cases.
+    """
+    client = zulip.Client(email=email, api_key=api_key,
+                          site=site,
+                          client="ZulipMercurial/" + VERSION)
+
+    message_data = {
+        "type": "stream",
+        "to": stream,
+        "subject": subject,
+        "content": content,
+    }
+
+    client.send_message(message_data)
+
+def get_config(ui, item):
+    try:
+        # configlist returns everything in lists.
+        return ui.configlist('zulip', item)[0]
+    except IndexError:
+        return None
+
+def hook(ui, repo, **kwargs):
+    """
+    Invoked by configuring a [hook] entry in .hg/hgrc.
+    """
+    hooktype = kwargs["hooktype"]
+    node = kwargs["node"]
+
+    ui.debug("Zulip: received {hooktype} event\n".format(hooktype=hooktype))
+
+    if hooktype != "changegroup":
+        ui.warn("Zulip: {hooktype} not supported\n".format(hooktype=hooktype))
+        exit(1)
+
+    ctx = repo.changectx(node)
+    branch = ctx.branch()
+
+    # If `branches` isn't specified, notify on all branches.
+    branch_whitelist = get_config(ui, "branches")
+    branch_blacklist = get_config(ui, "ignore_branches")
+
+    if branch_whitelist:
+        # Only send notifications on branches we are watching.
+        watched_branches = [b.lower().strip() for b in branch_whitelist.split(",")]
+        if branch.lower() not in watched_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    if branch_blacklist:
+        # Don't send notifications for branches we've ignored.
+        ignored_branches = [b.lower().strip() for b in branch_blacklist.split(",")]
+        if branch.lower() in ignored_branches:
+            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
+            exit(0)
+
+    # The first and final commits in the changeset.
+    base = repo[node].rev()
+    tip = len(repo)
+
+    email = get_config(ui, "email")
+    api_key = get_config(ui, "api_key")
+    site = get_config(ui, "site")
+
+    if not (email and api_key):
+        ui.warn("Zulip: missing email or api_key configurations\n")
+        ui.warn("in the [zulip] section of your .hg/hgrc.\n")
+        exit(1)
+
+    stream = get_config(ui, "stream")
+    # Give a default stream if one isn't provided.
+    if not stream:
+        stream = "commits"
+
+    web_url = get_config(ui, "web_url")
+    user = ctx.user()
+    content = format_summary_line(web_url, user, base, tip, branch, node)
+    content += format_commit_lines(web_url, repo, base, tip)
+
+    subject = branch
+
+    ui.debug("Sending to Zulip:\n")
+    ui.debug(content + "\n")
+
+    send_zulip(email, api_key, site, stream, subject, content)
--- a/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
+++ b/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
@@ -0,0 +1,149 @@
+/*
+* Copyright (c) 2014 Zulip, Inc
+*/
+
+package org.zulip.jira
+
+import static com.atlassian.jira.event.type.EventType.*
+
+import com.atlassian.jira.event.issue.AbstractIssueEventListener
+import com.atlassian.jira.event.issue.IssueEvent
+
+import java.util.logging.Level
+import java.util.logging.Logger
+
+import org.apache.commons.httpclient.HttpClient
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.methods.PostMethod
+import org.apache.commons.httpclient.NameValuePair
+
+class ZulipListener extends AbstractIssueEventListener {
+    Logger LOGGER = Logger.getLogger(ZulipListener.class.getName());
+
+    // The email address of one of the bots you created on your Zulip settings page.
+    String zulipEmail = ""
+    // That bot's API key.
+    String zulipAPIKey = ""
+
+    // What stream to send messages to. Must already exist.
+    String zulipStream = "jira"
+
+    // The base JIRA url for browsing
+    String issueBaseUrl = "https://jira.COMPANY.com/browse/"
+
+    // Your zulip domain, only change if you have a custom one
+    String base_url = "https://api.zulip.com"
+
+    @Override
+    void workflowEvent(IssueEvent event) {
+      processIssueEvent(event)
+    }
+
+    String processIssueEvent(IssueEvent event) {
+      String author = event.user.displayName
+      String issueId = event.issue.key
+      String issueUrl = issueBaseUrl + issueId
+      String issueUrlMd = String.format("[%s](%s)", issueId, issueBaseUrl + issueId)
+      String title = event.issue.summary
+      String subject = truncate(String.format("%s: %s", issueId, title), 60)
+      String assignee = "no one"
+      if (event.issue.assignee) {
+        assignee = event.issue.assignee.name
+      }
+      String comment = "";
+      if (event.comment) {
+        comment = event.comment.body
+      }
+
+      String content;
+
+      // Event types:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/event/type/EventType.html
+      // Issue API:
+      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/issue/Issue.html
+      switch (event.getEventTypeId()) {
+        case ISSUE_COMMENTED_ID:
+          content = String.format("%s **updated** %s with comment:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        case ISSUE_CREATED_ID:
+          content = String.format("%s **created** %s priority %s, assigned to @**%s**: \n\n> %s",
+                                  author, issueUrlMd, event.issue.priorityObject.name,
+                                  assignee, title)
+          break
+        case ISSUE_ASSIGNED_ID:
+          content = String.format("%s **reassigned** %s to **%s**",
+                                  author, issueUrlMd, assignee)
+          break
+        case ISSUE_DELETED_ID:
+          content = String.format("%s **deleted** %s!",
+                                  author, issueUrlMd)
+          break
+        case ISSUE_RESOLVED_ID:
+          content = String.format("%s **resolved** %s as %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_CLOSED_ID:
+          content = String.format("%s **closed** %s with resolution %s:\n\n> %s",
+                                  author, issueUrlMd, event.issue.resolutionObject.name,
+                                  comment)
+          break
+        case ISSUE_REOPENED_ID:
+          content = String.format("%s **reopened** %s:\n\n> %s",
+                                  author, issueUrlMd, comment)
+          break
+        default:
+          return
+      }
+
+      sendStreamMessage(zulipStream, subject, content)
+    }
+
+    String post(String method, NameValuePair[] parameters) {
+      PostMethod post = new PostMethod(zulipUrl(method))
+      post.setRequestHeader("Content-Type", post.FORM_URL_ENCODED_CONTENT_TYPE)
+      // TODO: Include more useful data in the User-agent
+      post.setRequestHeader("User-agent", "ZulipJira/0.1")
+      try {
+        post.setRequestBody(parameters)
+        HttpClient client = new HttpClient()
+        client.executeMethod(post)
+        String response = post.getResponseBodyAsString()
+        if (post.getStatusCode() != HttpStatus.SC_OK) {
+          String params = ""
+          for (NameValuePair pair: parameters) {
+              params += "\n" + pair.getName() + ":" + pair.getValue()
+          }
+          LOGGER.log(Level.SEVERE, "Error sending Zulip message:\n" + response + "\n\n" +
+                                   "We sent:" + params)
+        }
+        return response;
+      } catch (IOException e) {
+        throw new RuntimeException(e)
+      } finally {
+        post.releaseConnection()
+      }
+    }
+
+    String truncate(String string, int length) {
+      if (string.length() < length) {
+        return string
+      }
+      return string.substring(0, length - 3) + "..."
+    }
+
+    String sendStreamMessage(String stream, String subject, String message) {
+      NameValuePair[] body = [new NameValuePair("api-key", zulipAPIKey),
+                              new NameValuePair("email",   zulipEmail),
+                              new NameValuePair("type",    "stream"),
+                              new NameValuePair("to",      stream),
+                              new NameValuePair("subject", subject),
+                              new NameValuePair("content", message)]
+      return post("send_message", body);
+    }
+
+    String zulipUrl(method) {
+      return base_url + "/v1/" + method
+    }
+}
--- a/api/integrations/nagios/nagios-notify-zulip
+++ b/api/integrations/nagios/nagios-notify-zulip
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+import optparse
+import zulip
+
+VERSION = "0.9"
+# Nagios passes the notification details as command line options.
+# In Nagios, "output" means "first line of output", and "long
+# output" means "other lines of output".
+parser = optparse.OptionParser()
+parser.add_option('--output',      default='')
+parser.add_option('--long-output', default='')
+parser.add_option('--stream', default='nagios')
+parser.add_option('--config', default='/etc/nagios3/zuliprc')
+for opt in ('type', 'host', 'service', 'state'):
+    parser.add_option('--' + opt)
+(opts, args) = parser.parse_args()
+
+client = zulip.Client(config_file=opts.config, client="ZulipNagios/" + VERSION)
+
+msg = dict(type='stream', to=opts.stream)
+
+# Set a subject based on the host or service in question.  This enables
+# threaded discussion of multiple concurrent issues, and provides useful
+# context when narrowed.
+#
+# We send PROBLEM and RECOVERY messages to the same subject.
+if opts.service is None:
+    # Host notification
+    thing = 'host'
+    msg['subject'] = 'host %s' % (opts.host,)
+else:
+    # Service notification
+    thing = 'service'
+    msg['subject'] = 'service %s on %s' % (opts.service, opts.host)
+
+if len(msg['subject']) > 60:
+    msg['subject'] = msg['subject'][0:57].rstrip() + "..."
+# e.g. **PROBLEM**: service is CRITICAL
+msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
+
+# The "long output" can contain newlines represented by "\n" escape sequences.
+# The Nagios mail command uses /usr/bin/printf "%b" to expand these.
+# We will be more conservative and handle just this one escape sequence.
+output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip()
+if output:
+    # Put any command output in a code block.
+    msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
+
+client.send_message(msg)
--- a/api/integrations/nagios/zulip_nagios.cfg
+++ b/api/integrations/nagios/zulip_nagios.cfg
@@ -0,0 +1,21 @@
+define contact{
+        contact_name                    zulip
+        alias                           zulip
+        service_notification_period     24x7
+        host_notification_period        24x7
+        service_notification_options    w,u,c,r
+        host_notification_options       d,r
+        service_notification_commands   notify-service-by-zulip
+        host_notification_commands      notify-host-by-zulip
+        }
+
+# Zulip commands
+define command {
+    command_name    notify-host-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
+}
+
+define command {
+    command_name    notify-service-by-zulip
+    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
+}
--- a/api/integrations/nagios/zuliprc.example
+++ b/api/integrations/nagios/zuliprc.example
@@ -0,0 +1,5 @@
+# Fill these values in with the appropriate values for your realm, and
+# then install this value at /etc/nagios3/zuliprc
+[api]
+email = nagios-bot@example.com
+key = 0123456789abcdef0123456789abcdef
--- a/api/integrations/perforce/git_p4.py
+++ b/api/integrations/perforce/git_p4.py
--- a/api/integrations/perforce/license.txt
+++ b/api/integrations/perforce/license.txt
@@ -0,0 +1,26 @@
+git_p4.py was downloaded from https://raw.github.com/git/git/34022ba/git-p4.py
+
+The header of that file references <http://opensource.org/licenses/mit-license.php>,
+the textual contents of which are included below.
+
+The MIT License (MIT)
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
--- a/api/integrations/perforce/zulip_change-commit.py
+++ b/api/integrations/perforce/zulip_change-commit.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+'''Zulip notification change-commit hook.
+
+In Perforce, The "change-commit" trigger is fired after a metadata has been
+created, files have been transferred, and the changelist comitted to the depot
+database.
+
+This specific trigger expects command-line arguments in the form:
+  %change% %changeroot%
+
+For example:
+  1234 //depot/security/src/
+
+'''
+from __future__ import print_function
+
+import os
+import sys
+import os.path
+
+import git_p4
+
+__version__ = "0.1"
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_perforce_config as config
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipPerforce/" + __version__)
+
+try:
+    changelist = int(sys.argv[1])
+    changeroot = sys.argv[2]
+except IndexError:
+    print("Wrong number of arguments.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+except ValueError:
+    print("First argument must be an integer.\n\n", end=' ', file=sys.stderr)
+    print(__doc__, file=sys.stderr)
+    sys.exit(-1)
+
+metadata = git_p4.p4_describe(changelist)
+
+destination = config.commit_notice_destination(changeroot, changelist)
+if destination is None:
+    # Don't forward the notice anywhere
+    sys.exit(0)
+
+message = """**{0}** committed revision @{1} to `{2}`.
+
+> {3}
+""".format(metadata["user"], metadata["change"], changeroot, metadata["desc"])
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}
+client.send_message(message_data)
--- a/api/integrations/perforce/zulip_perforce_config.py
+++ b/api/integrations/perforce/zulip_perforce_config.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "p4-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the Perforce depot on the server
+# * changelist = the changelist id
+#
+# Returns a dictionary encoding the stream and topic to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for ones in the
+# "master-plan" and "secret" subdirectories of //depot/ to:
+# * stream "depot_subdirectory-commits"
+# * subject "change_root"
+def commit_notice_destination(path, changelist):
+    dirs = path.split('/')
+    if len(dirs) >= 4 and dirs[3] not in ("*", "..."):
+        directory = dirs[3]
+    else:
+        # No subdirectory, so just use "depot"
+        directory = dirs[2]
+
+    if directory not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "%s-commits" % (directory,),
+                    subject = path)
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# This should not need to change unless you have a custom Zulip subdomain.
+ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/rss/rss-bot
+++ b/api/integrations/rss/rss-bot
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# RSS integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import calendar
+import errno
+import hashlib
+from six.moves.html_parser import HTMLParser
+import logging
+import optparse
+import os
+import sys
+import time
+from six.moves import urllib
+
+import feedparser
+import zulip
+VERSION = "0.9"
+RSS_DATA_DIR = os.path.expanduser(os.path.join('~', '.cache', 'zulip-rss'))
+OLDNESS_THRESHOLD = 30 # days
+
+usage = """Usage: Send summaries of RSS entries for your favorite feeds to Zulip.
+
+This bot requires the feedparser module.
+
+To use this script:
+
+1. Create an RSS feed file containing 1 feed URL per line (default feed
+   file location: ~/.cache/zulip-rss/rss-feeds)
+2. Subscribe to the stream that will receive RSS updates (default stream: rss)
+3. create a ~/.zuliprc as described on https://zulip.com/api#api_keys
+4. Test the script by running it manually, like this:
+
+/usr/local/share/zulip/integrations/rss/rss-bot
+
+You can customize the location on the feed file and recipient stream, e.g.:
+
+/usr/local/share/zulip/integrations/rss/rss-bot --feed-file=/path/to/my-feeds --stream=my-rss-stream
+
+4. Configure a crontab entry for this script. A sample crontab entry for
+processing feeds stored in the default location and sending to the default
+stream every 5 minutes is:
+
+*/5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
+
+parser = optparse.OptionParser(usage)
+parser.add_option('--stream',
+                  dest='stream',
+                  help='The stream to which to send RSS messages.',
+                  default="rss",
+                  action='store')
+parser.add_option('--data-dir',
+                  dest='data_dir',
+                  help='The directory where feed metadata is stored',
+                  default=os.path.join(RSS_DATA_DIR),
+                  action='store')
+parser.add_option('--feed-file',
+                  dest='feed_file',
+                  help='The file containing a list of RSS feed URLs to follow, one URL per line',
+                  default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
+                  action='store')
+parser.add_option_group(zulip.generate_option_group(parser))
+(opts, args) = parser.parse_args()
+
+def mkdir_p(path):
+    # Python doesn't have an analog to `mkdir -p` < Python 3.2.
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+try:
+    mkdir_p(opts.data_dir)
+except OSError:
+    # We can't write to the logfile, so just print and give up.
+    print("Unable to store RSS data at %s." % (opts.data_dir,), file=sys.stderr)
+    exit(1)
+
+log_file = os.path.join(opts.data_dir, "rss-bot.log")
+log_format = "%(asctime)s: %(message)s"
+logging.basicConfig(format=log_format)
+
+formatter = logging.Formatter(log_format)
+file_handler = logging.FileHandler(log_file)
+file_handler.setFormatter(formatter)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(file_handler)
+
+def log_error_and_exit(error):
+    logger.error(error)
+    logger.error(usage)
+    exit(1)
+
+class MLStripper(HTMLParser):
+    def __init__(self):
+        self.reset()
+        self.fed = []
+
+    def handle_data(self, data):
+        self.fed.append(data)
+
+    def get_data(self):
+        return ''.join(self.fed)
+
+def strip_tags(html):
+    stripper = MLStripper()
+    stripper.feed(html)
+    return stripper.get_data()
+
+def compute_entry_hash(entry):
+    entry_time = entry.get("published", entry.get("updated"))
+    entry_id = entry.get("id", entry.get("link"))
+    return hashlib.md5(entry_id + str(entry_time)).hexdigest()
+
+def elide_subject(subject):
+    MAX_TOPIC_LENGTH = 60
+    if len(subject) > MAX_TOPIC_LENGTH:
+        subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
+    return subject
+
+def send_zulip(entry, feed_name):
+    content = "**[%s](%s)**\n%s\n%s" % (entry.title,
+                                  entry.link,
+                                  strip_tags(entry.summary),
+                                  entry.link)
+    message = {"type": "stream",
+               "sender": opts.email,
+               "to": opts.stream,
+               "subject": elide_subject(feed_name),
+               "content": content,
+               }
+    return client.send_message(message)
+
+try:
+    with open(opts.feed_file, "r") as f:
+        feed_urls = [feed.strip() for feed in f.readlines()]
+except IOError:
+    log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
+
+client = zulip.Client(email=opts.email, api_key=opts.api_key,
+                       site=opts.site, client="ZulipRSS/" + VERSION)
+
+first_message = True
+
+for feed_url in feed_urls:
+    feed_file = os.path.join(opts.data_dir, urllib.parse.urlparse(feed_url).netloc)
+
+    try:
+        with open(feed_file, "r") as f:
+            old_feed_hashes = dict((line.strip(), True) for line in f.readlines())
+    except IOError:
+        old_feed_hashes = {}
+
+    new_hashes = []
+    data = feedparser.parse(feed_url)
+
+    for entry in data.entries:
+        entry_hash = compute_entry_hash(entry)
+        # An entry has either been published or updated.
+        entry_time  = entry.get("published_parsed", entry.get("updated_parsed"))
+        if entry_time is not None and (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
+            # As a safeguard against misbehaving feeds, don't try to process
+            # entries older than some threshold.
+            continue
+        if entry_hash in old_feed_hashes:
+            # We've already seen this. No need to process any older entries.
+            break
+        if (not old_feed_hashes) and (len(new_hashes) >= 3):
+            # On a first run, pick up the 3 most recent entries. An RSS feed has
+            # entries in reverse chronological order.
+            break
+
+        feed_name = data.feed.title or feed_url
+
+        response = send_zulip(entry, feed_name)
+        if response["result"] != "success":
+            logger.error("Error processing %s" % (feed_url,))
+            logger.error(response)
+            if first_message:
+                # This is probably some fundamental problem like the stream not
+                # existing or something being misconfigured, so bail instead of
+                # getting the same error for every RSS entry.
+                log_error_and_exit("Failed to process first message")
+        # Go ahead and move on -- perhaps this entry is corrupt.
+        new_hashes.append(entry_hash)
+        first_message = False
+
+    with open(feed_file, "a") as f:
+        for hash in new_hashes:
+            f.write(hash + "\n")
+
+    logger.info("Sent zulips for %d %s entries" % (len(new_hashes), feed_url))
--- a/api/integrations/svn/post-commit
+++ b/api/integrations/svn/post-commit
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Zulip notification post-commit hook.
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "post-commit" script is run after a transaction is completed and a new
+# revision is created.  It is passed arguments on the command line in this
+# form:
+#  <path> <revision>
+# For example:
+# /srv/svn/carols 1843
+
+import os
+import sys
+import os.path
+import pysvn
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_svn_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipSVN/" + VERSION)
+svn = pysvn.Client()
+
+path, rev = sys.argv[1:]
+
+# since its a local path, prepend "file://"
+path = "file://" + path
+
+entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0]
+message = """**{0}** committed revision r{1} to `{2}`.
+
+> {3}
+""".format(entry['author'], rev, path.split('/')[-1], entry['revprops']['svn:log'])
+
+destination = config.commit_notice_destination(path, rev)
+
+message_data = {
+    "type": "stream",
+    "to": destination["stream"],
+    "subject": destination["subject"],
+    "content": message,
+}
+client.send_message(message_data)
--- a/api/integrations/svn/zulip_svn_config.py
+++ b/api/integrations/svn/zulip_svn_config.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Change these values to configure authentication for the plugin
+ZULIP_USER = "svn-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# commit_notice_destination() lets you customize where commit notices
+# are sent to with the full power of a Python function.
+#
+# It takes the following arguments:
+# * path   = the path to the svn repository on the server
+# * commit = the commit id
+#
+# Returns a dictionary encoding the stream and subject to send the
+# notification to (or None to send no notification).
+#
+# The default code below will send every commit except for the "evil-master-plan"
+# and "my-super-secret-repository" repos to
+# * stream "commits"
+# * topic "branch_name"
+def commit_notice_destination(path, commit):
+    repo = path.split('/')[-1]
+    if repo not in ["evil-master-plan", "my-super-secret-repository"]:
+        return dict(stream  = "commits",
+                    subject = u"%s" % (repo,))
+
+    # Return None for cases where you don't want a notice sent
+    return None
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip server's API URI
+ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/trac/zulip_trac.py
+++ b/api/integrations/trac/zulip_trac.py
@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Zulip trac plugin -- sends zulips when tickets change.
+#
+# Install by copying this file and zulip_trac_config.py to the trac
+# plugins/ subdirectory, customizing the constants in
+# zulip_trac_config.py, and then adding "zulip_trac" to the
+# components section of the conf/trac.ini file, like so:
+#
+# [components]
+# zulip_trac = enabled
+#
+# You may then need to restart trac (or restart Apache) for the bot
+# (or changes to the bot) to actually be loaded by trac.
+
+from trac.core import Component, implements
+from trac.ticket import ITicketChangeListener
+import sys
+import os.path
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_trac_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+
+import zulip
+client = zulip.Client(
+    email=config.ZULIP_USER,
+    site=config.ZULIP_SITE,
+    api_key=config.ZULIP_API_KEY,
+    client="ZulipTrac/" + VERSION)
+
+def markdown_ticket_url(ticket, heading="ticket"):
+    return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
+
+def markdown_block(desc):
+    return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
+
+def truncate(string, length):
+    if len(string) <= length:
+        return string
+    return string[:length - 3] + "..."
+
+def trac_subject(ticket):
+    return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
+
+def send_update(ticket, content):
+    client.send_message({
+            "type": "stream",
+            "to": config.STREAM_FOR_NOTIFICATIONS,
+            "content": content,
+            "subject": trac_subject(ticket)
+            })
+
+class ZulipPlugin(Component):
+    implements(ITicketChangeListener)
+
+    def ticket_created(self, ticket):
+        """Called when a ticket is created."""
+        content = "%s created %s in component **%s**, priority **%s**:\n" % \
+            (ticket.values.get("reporter"), markdown_ticket_url(ticket),
+             ticket.values.get("component"), ticket.values.get("priority"))
+        # Include the full subject if it will be truncated
+        if len(ticket.values.get("summary")) > 60:
+            content += "**%s**\n" % (ticket.values.get("summary"),)
+        if ticket.values.get("description") != "":
+            content += "%s" % (markdown_block(ticket.values.get("description")),)
+        send_update(ticket, content)
+
+    def ticket_changed(self, ticket, comment, author, old_values):
+        """Called when a ticket is modified.
+
+        `old_values` is a dictionary containing the previous values of the
+        fields that have changed.
+        """
+        if not (set(old_values.keys()).intersection(set(config.TRAC_NOTIFY_FIELDS)) or
+                (comment and "comment" in set(config.TRAC_NOTIFY_FIELDS))):
+            return
+
+        content = "%s updated %s" % (author, markdown_ticket_url(ticket))
+        if comment:
+            content += ' with comment: %s\n\n' % (markdown_block(comment),)
+        else:
+            content += ":\n\n"
+        field_changes = []
+        for key in old_values.keys():
+            if key == "description":
+                content += '- Changed %s from %s\n\nto %s' % (key, markdown_block(old_values.get(key)),
+                                                              markdown_block(ticket.values.get(key)))
+            elif old_values.get(key) == "":
+                field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
+            elif ticket.values.get(key) == "":
+                field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
+            else:
+                field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
+                                                               ticket.values.get(key)))
+        content += ", ".join(field_changes)
+
+        send_update(ticket, content)
+
+    def ticket_deleted(self, ticket):
+        """Called when a ticket is deleted."""
+        content = "%s was deleted." % markdown_ticket_url(ticket, heading="Ticket")
+        send_update(ticket, content)
--- a/api/integrations/trac/zulip_trac_config.py
+++ b/api/integrations/trac/zulip_trac_config.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# See zulip_trac.py for installation and configuration instructions
+
+# Change these constants to configure the plugin:
+ZULIP_USER = "trac-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+STREAM_FOR_NOTIFICATIONS = "trac"
+TRAC_BASE_TICKET_URL = "https://trac.example.com/ticket"
+
+# Most people find that having every change in Trac result in a
+# notification is too noisy -- in particular, when someone goes
+# through recategorizing a bunch of tickets, that can often be noisy
+# and annoying.  We solve this issue by only sending a notification
+# for changes to the fields listed below.
+#
+# TRAC_NOTIFY_FIELDS lets you specify which fields will trigger a
+# Zulip notification in response to a trac update; you should change
+# this list to match your team's workflow.  The complete list of
+# possible fields is:
+#
+# (priority, milestone, cc, owner, keywords, component, severity,
+#  type, versions, description, resolution, summary, comment)
+TRAC_NOTIFY_FIELDS = ["description", "summary", "resolution", "comment", "owner"]
+
+## If properly installed, the Zulip API should be in your import
+## path, but if not, set a custom path below
+ZULIP_API_PATH = None
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/twitter/twitter-bot
+++ b/api/integrations/twitter/twitter-bot
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Twitter integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import os
+import sys
+import optparse
+import six.moves.configparser
+
+import zulip
+VERSION = "0.9"
+CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
+
+def write_config(config, since_id, user):
+    config.set('twitter', 'since_id', since_id)
+    config.set('twitter', 'user_id', user)
+    with open(CONFIGFILE, 'wb') as configfile:
+        config.write(configfile)
+
+parser = optparse.OptionParser(r"""
+
+%prog --user foo@example.com --api-key 0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --twitter-id twitter_handle
+
+    Slurp tweets on your timeline into a specific zulip stream.
+
+    Run this on your personal machine.  Your API key and twitter id
+    are revealed to local users through the command line or config
+    file.
+
+    This bot uses OAuth to authenticate with twitter. Please create a
+    ~/.zulip_twitterrc with the following contents:
+
+    [twitter]
+    consumer_key =
+    consumer_secret =
+    access_token_key =
+    access_token_secret =
+
+    In order to obtain a consumer key & secret, you must register a
+    new application under your twitter account:
+
+    1. Go to http://dev.twitter.com
+    2. Log in
+    3. In the menu under your username, click My Applications
+    4. Create a new application
+
+    Make sure to go the application you created and click "create my
+    access token" as well.  Fill in the values displayed.
+
+    Depends on: twitter-python
+""")
+
+parser.add_option('--twitter-id',
+    help='Twitter username to poll for new tweets from"',
+    metavar='URL')
+parser.add_option('--stream',
+    help='Default zulip stream to write tweets to')
+parser.add_option('--limit-tweets',
+    default=15,
+    type='int',
+    help='Maximum number of tweets to push at once')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if not options.twitter_id:
+    parser.error('You must specify --twitter-id')
+
+try:
+    config = six.moves.configparser.ConfigParser()
+    config.read(CONFIGFILE)
+
+    consumer_key = config.get('twitter', 'consumer_key')
+    consumer_secret = config.get('twitter', 'consumer_secret')
+    access_token_key = config.get('twitter', 'access_token_key')
+    access_token_secret = config.get('twitter', 'access_token_secret')
+except (six.moves.configparser.NoSectionError, six.moves.configparser.NoOptionError):
+   parser.error("Please provide a ~/.zulip_twitterrc")
+
+if not consumer_key or not consumer_secret or not access_token_key or not access_token_secret:
+   parser.error("Please provide a ~/.zulip_twitterrc")
+
+try:
+    import twitter
+except ImportError:
+    parser.error("Please install twitter-python")
+
+api = twitter.Api(consumer_key=consumer_key,
+                  consumer_secret=consumer_secret,
+                  access_token_key=access_token_key,
+                  access_token_secret=access_token_secret)
+
+
+user = api.VerifyCredentials()
+
+if not user.GetId():
+    print("Unable to log in to twitter with supplied credentials. Please double-check and try again")
+    sys.exit()
+
+try:
+    since_id = config.getint('twitter', 'since_id')
+except six.moves.configparser.NoOptionError:
+    since_id = -1
+
+try:
+    user_id = config.get('twitter', 'user_id')
+except six.moves.configparser.NoOptionError:
+    user_id = options.twitter_id
+
+client = zulip.Client(
+    email=options.zulip_email,
+    api_key=options.zulip_api_key,
+    site=options.zulip_site,
+    client="ZulipTwitter/" + VERSION,
+    verbose=True)
+
+if since_id < 0 or options.twitter_id != user_id:
+    # No since id yet, fetch the latest and then start monitoring from next time
+    # Or, a different user id is being asked for, so start from scratch
+    # Either way, fetch last 5 tweets to start off
+    statuses = api.GetFriendsTimeline(user=options.twitter_id, count=5)
+else:
+    # We have a saved last id, so insert all newer tweets into the zulip stream
+    statuses = api.GetFriendsTimeline(user=options.twitter_id, since_id=since_id)
+
+for status in statuses[::-1][:options.limit_tweets]:
+    composed = "%s (%s)" % (status.GetUser().GetName(), status.GetUser().GetScreenName())
+    message = {
+      "type": "stream",
+      "to": [options.stream],
+      "subject": composed,
+      "content": status.GetText(),
+    }
+
+    ret = client.send_message(message)
+
+    if ret['result'] == 'error':
+        # If sending failed (e.g. no such stream), abort and retry next time
+        print("Error sending message to zulip: %s" % ret['msg'])
+        break
+    else:
+        since_id = status.GetId()
+
+write_config(config, since_id, user_id)
--- a/api/integrations/twitter/twitter-search-bot
+++ b/api/integrations/twitter/twitter-search-bot
@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Twitter search integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import os
+import sys
+import optparse
+import six.moves.configparser
+
+import zulip
+VERSION = "0.9"
+CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
+
+def write_config(config, since_id):
+    if 'search' not in config.sections():
+        config.add_section('search')
+    config.set('search', 'since_id', since_id)
+    with open(CONFIGFILE, 'wb') as configfile:
+        config.write(configfile)
+
+parser = optparse.OptionParser(r"""
+
+%prog --user foo@zulip.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --search="@nprnews,quantum physics"
+
+Send Twitter search results to a Zulip stream.
+
+Depends on: twitter-python version 1.0 or later
+
+To use this script:
+
+1. Set up Twitter authentication, as described below
+2. Subscribe to the stream that will receive Twitter updates (default stream: twitter)
+3. Test the script by running it manually, like this:
+
+/usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
+
+4. Configure a crontab entry for this script. A sample crontab entry
+that will process tweets every 5 minutes is:
+
+*/5 * * * * /usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
+
+== Setting up Twitter authentications ==
+
+Run this on a personal or trusted machine, because your API key is
+visible to local users through the command line or config file.
+
+This bot uses OAuth to authenticate with Twitter. Please create a
+~/.zulip_twitterrc with the following contents:
+
+[twitter]
+consumer_key =
+consumer_secret =
+access_token_key =
+access_token_secret =
+
+In order to obtain a consumer key & secret, you must register a
+new application under your Twitter account:
+
+1. Go to http://dev.twitter.com
+2. Log in
+3. In the menu under your username, click My Applications
+4. Create a new application
+
+Make sure to go the application you created and click "create my
+access token" as well. Fill in the values displayed.
+""")
+
+parser.add_option('--search',
+                  dest='search_terms',
+                  help='Terms to search on',
+                  action='store')
+parser.add_option('--stream',
+                  dest='stream',
+                  help='The stream to which to send tweets',
+                  default="twitter",
+                  action='store')
+parser.add_option('--limit-tweets',
+                  default=15,
+                  type='int',
+                  help='Maximum number of tweets to send at once')
+
+parser.add_option_group(zulip.generate_option_group(parser))
+(opts, args) = parser.parse_args()
+
+if not opts.search_terms:
+    parser.error('You must specify a search term.')
+
+try:
+    config = six.moves.configparser.ConfigParser()
+    config.read(CONFIGFILE)
+
+    consumer_key = config.get('twitter', 'consumer_key')
+    consumer_secret = config.get('twitter', 'consumer_secret')
+    access_token_key = config.get('twitter', 'access_token_key')
+    access_token_secret = config.get('twitter', 'access_token_secret')
+except (six.moves.configparser.NoSectionError, six.moves.configparser.NoOptionError):
+   parser.error("Please provide a ~/.zulip_twitterrc")
+
+if not (consumer_key and consumer_secret and access_token_key and access_token_secret):
+   parser.error("Please provide a ~/.zulip_twitterrc")
+
+try:
+    since_id = config.getint('search', 'since_id')
+except (six.moves.configparser.NoOptionError, six.moves.configparser.NoSectionError):
+    since_id = 0
+
+try:
+    import twitter
+except ImportError:
+    parser.error("Please install twitter-python")
+
+api = twitter.Api(consumer_key=consumer_key,
+                  consumer_secret=consumer_secret,
+                  access_token_key=access_token_key,
+                  access_token_secret=access_token_secret)
+
+user = api.VerifyCredentials()
+
+if not user.GetId():
+    print("Unable to log in to Twitter with supplied credentials.\
+Please double-check and try again.")
+    sys.exit()
+
+client = zulip.Client(
+    email=opts.email,
+    api_key=opts.api_key,
+    site=opts.site,
+    client="ZulipTwitterSearch/" + VERSION,
+    verbose=True)
+
+search_query = " OR ".join(opts.search_terms.split(","))
+statuses = api.GetSearch(search_query, since_id=since_id)
+
+for status in statuses[::-1][:opts.limit_tweets]:
+    # https://twitter.com/eatevilpenguins/status/309995853408530432
+    composed = "%s (%s)" % (status.GetUser().GetName(),
+                            status.GetUser().GetScreenName())
+    url = "https://twitter.com/%s/status/%s" % (status.GetUser().GetScreenName(),
+                                                status.GetId())
+    content = status.GetText()
+
+    search_term_used = None
+    for term in opts.search_terms.split(","):
+        if term.lower() in content.lower():
+            search_term_used = term
+            break
+    # For some reason (perhaps encodings or message tranformations we
+    # didn't anticipate), we don't know what term was used, so use a
+    # default.
+    if not search_term_used:
+        search_term_used = "mentions"
+
+    message = {
+      "type": "stream",
+      "to": [opts.stream],
+      "subject": search_term_used,
+      "content": url,
+    }
+
+    ret = client.send_message(message)
+
+    if ret['result'] == 'error':
+        # If sending failed (e.g. no such stream), abort and retry next time
+        print("Error sending message to zulip: %s" % ret['msg'])
+        break
+    else:
+        since_id = status.GetId()
+
+write_config(config, since_id)
--- a/api/setup.cfg
+++ b/api/setup.cfg
@@ -0,0 +1,3 @@
+# This way our scripts are installed in the data directory
+[aliases]
+install = install_data install
--- a/api/setup.py
+++ b/api/setup.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+if False: from typing import Any, Generator, List, Tuple
+
+import os
+import sys
+
+import itertools
+
+def version():
+    # type: () -> str
+    version_py = os.path.join(os.path.dirname(__file__), "zulip", "__init__.py")
+    with open(version_py) as in_handle:
+        version_line = next(itertools.dropwhile(lambda x: not x.startswith("__version__"),
+                                           in_handle))
+    version = version_line.split('=')[-1].strip().replace('"', '')
+    return version
+
+def recur_expand(target_root, dir):
+    # type: (Any, Any) -> Generator[Tuple[str, List[str]], None, None]
+    for root, _, files in os.walk(dir):
+        paths = [os.path.join(root, f) for f in files]
+        if len(paths):
+            yield os.path.join(target_root, root), paths
+
+# We should be installable with either setuptools or distutils.
+package_info = dict(
+    name='zulip',
+    version=version(),
+    description='Bindings for the Zulip message API',
+    author='Zulip, Inc.',
+    author_email='zulip-devel@googlegroups.com',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Web Environment',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Topic :: Communications :: Chat',
+    ],
+    url='https://www.zulip.com/dist/api/',
+    packages=['zulip'],
+    data_files=[('share/zulip/examples', ["examples/zuliprc", "examples/send-message", "examples/subscribe",
+                                           "examples/get-public-streams", "examples/unsubscribe",
+                                           "examples/list-members", "examples/list-subscriptions",
+                                           "examples/print-messages", "examples/recent-messages"])] + \
+        list(recur_expand('share/zulip', 'integrations/')),
+    scripts=["bin/zulip-send"],
+) # type: Dict[str, Any]
+
+setuptools_info = dict(
+    install_requires=['requests>=0.12.1',
+                      'simplejson',
+                      'six',
+                      'typing',
+    ],
+)
+
+try:
+    from setuptools import setup
+    package_info.update(setuptools_info)
+except ImportError:
+    from distutils.core import setup
+    from distutils.version import LooseVersion
+    # Manual dependency check
+    try:
+        import simplejson
+    except ImportError:
+        print("simplejson is not installed", file=sys.stderr)
+        sys.exit(1)
+    try:
+        import requests
+        assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1')) # type: ignore # https://github.com/JukkaL/mypy/issues/1165
+    except (ImportError, AssertionError):
+        print("requests >=0.12.1 is not installed", file=sys.stderr)
+        sys.exit(1)
+
+
+setup(**package_info)
--- a/api/zulip/init.py
+++ b/api/zulip/init.py
@@ -0,0 +1,527 @@
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+import simplejson
+import requests
+import time
+import traceback
+import sys
+import os
+import optparse
+import platform
+import random
+from distutils.version import LooseVersion
+
+from six.moves.configparser import SafeConfigParser
+from six.moves import urllib
+import logging
+import six
+from typing import Any, Dict
+
+
+__version__ = "0.2.5"
+
+logger = logging.getLogger(__name__)
+
+# Check that we have a recent enough version
+# Older versions don't provide the 'json' attribute on responses.
+assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1')) # type: ignore # https://github.com/python/mypy/issues/1165 and https://github.com/python/typeshed/pull/206
+# In newer versions, the 'json' attribute is a function, not a property
+requests_json_is_function = callable(requests.Response.json)
+
+API_VERSTRING = "v1/"
+
+class CountingBackoff(object):
+    def __init__(self, maximum_retries=10, timeout_success_equivalent=None):
+        self.number_of_retries = 0
+        self.maximum_retries = maximum_retries
+        self.timeout_success_equivalent = timeout_success_equivalent
+        self.last_attempt_time = 0
+
+    def keep_going(self):
+        self._check_success_timeout()
+        return self.number_of_retries < self.maximum_retries
+
+    def succeed(self):
+        self.number_of_retries = 0
+        self.last_attempt_time = time.time()
+
+    def fail(self):
+        self._check_success_timeout()
+        self.number_of_retries = min(self.number_of_retries + 1,
+                                     self.maximum_retries)
+        self.last_attempt_time = time.time()
+
+    def _check_success_timeout(self):
+        if (self.timeout_success_equivalent is not None
+            and self.last_attempt_time != 0
+            and time.time() - self.last_attempt_time > self.timeout_success_equivalent):
+            self.number_of_retries = 0
+
+class RandomExponentialBackoff(CountingBackoff):
+    def fail(self):
+        super(RandomExponentialBackoff, self).fail()
+        # Exponential growth with ratio sqrt(2); compute random delay
+        # between x and 2x where x is growing exponentially
+        delay_scale = int(2 ** (self.number_of_retries / 2.0 - 1)) + 1
+        delay = delay_scale + random.randint(1, delay_scale)
+        message = "Sleeping for %ss [max %s] before retrying." % (delay, delay_scale * 2)
+        try:
+            logger.warning(message)
+        except NameError:
+            print(message)
+        time.sleep(delay)
+
+def _default_client():
+    return "ZulipPython/" + __version__
+
+def generate_option_group(parser, prefix=''):
+    group = optparse.OptionGroup(parser, 'Zulip API configuration')
+    group.add_option('--%ssite' % (prefix,),
+                     dest="zulip_site",
+                     help="Zulip server URI",
+                     default=None)
+    group.add_option('--%sapi-key' % (prefix,),
+                     dest="zulip_api_key",
+                     action='store')
+    group.add_option('--%suser' % (prefix,),
+                     dest='zulip_email',
+                     help='Email address of the calling bot or user.')
+    group.add_option('--%sconfig-file' % (prefix,),
+                     action='store',
+                     dest="zulip_config_file",
+                     help='Location of an ini file containing the\nabove information. (default ~/.zuliprc)')
+    group.add_option('-v', '--verbose',
+                     action='store_true',
+                     help='Provide detailed output.')
+    group.add_option('--%sclient' % (prefix,),
+                     action='store',
+                     default=None,
+                     dest="zulip_client",
+                     help=optparse.SUPPRESS_HELP)
+    group.add_option('--insecure',
+                     action='store_true',
+                     dest='insecure',
+                     help='''Do not verify the server certificate.
+                          The https connection will not be secure.''')
+    group.add_option('--cert-bundle',
+                     action='store',
+                     dest='cert_bundle',
+                     help='''Specify a file containing either the
+                          server certificate, or a set of trusted
+                          CA certificates. This will be used to
+                          verify the server's identity. All
+                          certificates should be PEM encoded.''')
+    group.add_option('--client-cert',
+                     action='store',
+                     dest='client_cert',
+                     help='''Specify a file containing a client
+                          certificate (not needed for most deployments).''')
+    group.add_option('--client-cert-key',
+                     action='store',
+                     dest='client_cert_key',
+                     help='''Specify a file containing the client
+                          certificate's key (if it is in a separate
+                          file).''')
+    return group
+
+def init_from_options(options, client=None):
+    if options.zulip_client is not None:
+        client = options.zulip_client
+    elif client is None:
+        client = _default_client()
+    return Client(email=options.zulip_email, api_key=options.zulip_api_key,
+                  config_file=options.zulip_config_file, verbose=options.verbose,
+                  site=options.zulip_site, client=client,
+                  cert_bundle=options.cert_bundle, insecure=options.insecure,
+                  client_cert=options.client_cert,
+                  client_cert_key=options.client_cert_key)
+
+def get_default_config_filename():
+    config_file = os.path.join(os.environ["HOME"], ".zuliprc")
+    if (not os.path.exists(config_file) and
+        os.path.exists(os.path.join(os.environ["HOME"], ".humbugrc"))):
+        raise RuntimeError("The Zulip API configuration file is now ~/.zuliprc; please run:\n\n"
+                           "  mv ~/.humbugrc ~/.zuliprc\n")
+    return config_file
+
+class Client(object):
+    def __init__(self, email=None, api_key=None, config_file=None,
+                 verbose=False, retry_on_errors=True,
+                 site=None, client=None,
+                 cert_bundle=None, insecure=None,
+                 client_cert=None, client_cert_key=None):
+        if client is None:
+            client = _default_client()
+
+        if config_file is None:
+            config_file = get_default_config_filename()
+        if os.path.exists(config_file):
+            config = SafeConfigParser()
+            with open(config_file, 'r') as f:
+                config.readfp(f, config_file)
+            if api_key is None:
+                api_key = config.get("api", "key")
+            if email is None:
+                email = config.get("api", "email")
+            if site is None and config.has_option("api", "site"):
+                site = config.get("api", "site")
+            if client_cert is None and config.has_option("api", "client_cert"):
+                client_cert = config.get("api", "client_cert")
+            if client_cert_key is None and config.has_option("api", "client_cert_key"):
+                client_cert_key = config.get("api", "client_cert_key")
+            if cert_bundle is None and config.has_option("api", "cert_bundle"):
+                cert_bundle = config.get("api", "cert_bundle")
+            if insecure is None and config.has_option("api", "insecure"):
+                # Be quite strict about what is accepted so that users don't
+                # disable security unintentionally.
+                insecure_setting = config.get("api", "insecure").lower()
+                if insecure_setting == "true":
+                    insecure = True
+                elif insecure_setting == "false":
+                    insecure = False
+                else:
+                    raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
+                                       % (insecure_setting, config_file))
+        elif None in (api_key, email):
+            raise RuntimeError("api_key or email not specified and %s does not exist"
+                               % (config_file,))
+
+        self.api_key = api_key
+        self.email = email
+        self.verbose = verbose
+        if site is not None:
+            if not site.startswith("http"):
+                site = "https://" + site
+            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
+            site = site.rstrip("/")
+            self.base_url = site
+        else:
+            self.base_url = "https://api.zulip.com"
+
+        if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
+            self.base_url += "/api"
+        self.base_url += "/"
+        self.retry_on_errors = retry_on_errors
+        self.client_name = client
+
+        if insecure:
+            self.tls_verification=False
+        elif cert_bundle is not None:
+            if not os.path.isfile(cert_bundle):
+                raise RuntimeError("tls bundle '%s' does not exist"
+                                   %(cert_bundle,))
+            self.tls_verification=cert_bundle
+        else:
+            # Default behavior: verify against system CA certificates
+            self.tls_verification=True
+
+        if client_cert is None:
+            if client_cert_key is not None:
+                raise RuntimeError("client cert key '%s' specified, but no client cert public part provided"
+                                   %(client_cert_key,))
+        else: # we have a client cert
+            if not os.path.isfile(client_cert):
+                raise RuntimeError("client cert '%s' does not exist"
+                                   %(client_cert,))
+            if client_cert_key is not None:
+                if not os.path.isfile(client_cert_key):
+                    raise RuntimeError("client cert key '%s' does not exist"
+                                       %(client_cert_key,))
+        self.client_cert = client_cert
+        self.client_cert_key = client_cert_key
+
+    def get_user_agent(self):
+        vendor = ''
+        vendor_version = ''
+        try:
+            vendor = platform.system()
+            vendor_version = platform.release()
+        except IOError:
+            # If the calling process is handling SIGCHLD, platform.system() can
+            # fail with an IOError.  See http://bugs.python.org/issue9127
+            pass
+
+        if vendor == "Linux":
+            vendor, vendor_version, dummy = platform.linux_distribution()
+        elif vendor == "Windows":
+            vendor_version = platform.win32_ver()[1]
+        elif vendor == "Darwin":
+            vendor_version = platform.mac_ver()[0]
+
+        return "{client_name} ({vendor}; {vendor_version})".format(
+                client_name=self.client_name,
+                vendor=vendor,
+                vendor_version=vendor_version,
+                )
+
+    def do_api_query(self, orig_request, url, method="POST", longpolling = False):
+        request = {}
+
+        for (key, val) in six.iteritems(orig_request):
+            if isinstance(val, str) or isinstance(val, six.text_type):
+                request[key] = val
+            else:
+                request[key] = simplejson.dumps(val)
+
+        query_state = {
+            'had_error_retry': False,
+            'request': request,
+            'failures': 0,
+        } # type: Dict[str, Any]
+
+        def error_retry(error_string):
+            if not self.retry_on_errors or query_state["failures"] >= 10:
+                return False
+            if self.verbose:
+                if not query_state["had_error_retry"]:
+                    sys.stdout.write("zulip API(%s): connection error%s -- retrying." % \
+                            (url.split(API_VERSTRING, 2)[0], error_string,))
+                    query_state["had_error_retry"] = True
+                else:
+                    sys.stdout.write(".")
+                sys.stdout.flush()
+            query_state["request"]["dont_block"] = simplejson.dumps(True)
+            time.sleep(1)
+            query_state["failures"] += 1
+            return True
+
+        def end_error_retry(succeeded):
+            if query_state["had_error_retry"] and self.verbose:
+                if succeeded:
+                    print("Success!")
+                else:
+                    print("Failed!")
+
+        while True:
+            try:
+                if method == "GET":
+                    kwarg = "params"
+                else:
+                    kwarg = "data"
+                kwargs = {kwarg: query_state["request"]}
+
+                # Build a client cert object for requests
+                if self.client_cert_key is not None:
+                    client_cert = (self.client_cert, self.client_cert_key)
+                else:
+                    client_cert = self.client_cert
+
+                res = requests.request(
+                        method,
+                        urllib.parse.urljoin(self.base_url, url),
+                        auth=requests.auth.HTTPBasicAuth(self.email,
+                                                         self.api_key),
+                        verify=self.tls_verification,
+                        cert=client_cert,
+                        timeout=90,
+                        headers={"User-agent": self.get_user_agent()},
+                        **kwargs)
+
+                # On 50x errors, try again after a short sleep
+                if str(res.status_code).startswith('5'):
+                    if error_retry(" (server %s)" % (res.status_code,)):
+                        continue
+                    # Otherwise fall through and process the python-requests error normally
+            except (requests.exceptions.Timeout, requests.exceptions.SSLError) as e:
+                # Timeouts are either a Timeout or an SSLError; we
+                # want the later exception handlers to deal with any
+                # non-timeout other SSLErrors
+                if (isinstance(e, requests.exceptions.SSLError) and
+                    str(e) != "The read operation timed out"):
+                    raise
+                if longpolling:
+                    # When longpolling, we expect the timeout to fire,
+                    # and the correct response is to just retry
+                    continue
+                else:
+                    end_error_retry(False)
+                    return {'msg': "Connection error:\n%s" % traceback.format_exc(),
+                            "result": "connection-error"}
+            except requests.exceptions.ConnectionError:
+                if error_retry(""):
+                    continue
+                end_error_retry(False)
+                return {'msg': "Connection error:\n%s" % traceback.format_exc(),
+                        "result": "connection-error"}
+            except Exception:
+                # We'll split this out into more cases as we encounter new bugs.
+                return {'msg': "Unexpected error:\n%s" % traceback.format_exc(),
+                        "result": "unexpected-error"}
+
+            try:
+                if requests_json_is_function:
+                    json_result = res.json()
+                else:
+                    json_result = res.json
+            except Exception:
+                json_result = None
+
+            if json_result is not None:
+                end_error_retry(True)
+                return json_result
+            end_error_retry(False)
+            return {'msg': "Unexpected error from the server", "result": "http-error",
+                    "status_code": res.status_code}
+
+    @classmethod
+    def _register(cls, name, url=None, make_request=None,
+                  method="POST", computed_url=None, **query_kwargs):
+        if url is None:
+            url = name
+        if make_request is None:
+            def make_request(request=None):
+                if request is None:
+                    request = {}
+                return request
+        def call(self, *args, **kwargs):
+            request = make_request(*args, **kwargs)
+            if computed_url is not None:
+                req_url = computed_url(request)
+            else:
+                req_url = url
+            return self.do_api_query(request, API_VERSTRING + req_url, method=method, **query_kwargs)
+        call.__name__ = name
+        setattr(cls, name, call)
+
+    def call_on_each_event(self, callback, event_types=None, narrow=None):
+        if narrow is None:
+            narrow = []
+        def do_register():
+            while True:
+                if event_types is None:
+                    res = self.register()
+                else:
+                    res = self.register(event_types=event_types, narrow=narrow)
+
+                if 'error' in res.get('result'):
+                    if self.verbose:
+                        print("Server returned error:\n%s" % res['msg'])
+                    time.sleep(1)
+                else:
+                    return (res['queue_id'], res['last_event_id'])
+
+        queue_id = None
+        while True:
+            if queue_id is None:
+                (queue_id, last_event_id) = do_register()
+
+            res = self.get_events(queue_id=queue_id, last_event_id=last_event_id)
+            if 'error' in res.get('result'):
+                if res["result"] == "http-error":
+                    if self.verbose:
+                        print("HTTP error fetching events -- probably a server restart")
+                elif res["result"] == "connection-error":
+                    if self.verbose:
+                        print("Connection error fetching events -- probably server is temporarily down?")
+                else:
+                    if self.verbose:
+                        print("Server returned error:\n%s" % res["msg"])
+                    if res["msg"].startswith("Bad event queue id:"):
+                        # Our event queue went away, probably because
+                        # we were asleep or the server restarted
+                        # abnormally.  We may have missed some
+                        # events while the network was down or
+                        # something, but there's not really anything
+                        # we can do about it other than resuming
+                        # getting new ones.
+                        #
+                        # Reset queue_id to register a new event queue.
+                        queue_id = None
+                # TODO: Make this back off once it's more reliable
+                time.sleep(1)
+                continue
+
+            for event in res['events']:
+                last_event_id = max(last_event_id, int(event['id']))
+                callback(event)
+
+    def call_on_each_message(self, callback):
+        def event_callback(event):
+            if event['type'] == 'message':
+                callback(event['message'])
+
+        self.call_on_each_event(event_callback, ['message'])
+
+def _mk_subs(streams, **kwargs):
+    result = kwargs
+    result['subscriptions'] = streams
+    return result
+
+def _mk_rm_subs(streams):
+    return {'delete': streams}
+
+def _mk_deregister(queue_id):
+    return {'queue_id': queue_id}
+
+def _mk_events(event_types=None, narrow=None):
+    if event_types is None:
+        return dict()
+    if narrow is None:
+        narrow = []
+    return dict(event_types=event_types, narrow=narrow)
+
+def _kwargs_to_dict(**kwargs):
+    return kwargs
+
+class ZulipStream(object):
+    """
+    A Zulip stream-like object
+    """
+
+    def __init__(self, type, to, subject, **kwargs):
+        self.client = Client(**kwargs)
+        self.type = type
+        self.to = to
+        self.subject = subject
+
+    def write(self, content):
+        message = {"type": self.type,
+                   "to": self.to,
+                   "subject": self.subject,
+                   "content": content}
+        self.client.send_message(message)
+
+    def flush(self):
+        pass
+
+Client._register('send_message', url='messages', make_request=(lambda request: request))
+Client._register('update_message', method='PATCH', url='messages', make_request=(lambda request: request))
+Client._register('get_messages', method='GET', url='messages/latest', longpolling=True)
+Client._register('get_events', url='events', method='GET', longpolling=True, make_request=(lambda **kwargs: kwargs))
+Client._register('register', make_request=_mk_events)
+Client._register('export', method='GET', url='export')
+Client._register('deregister', url="events", method="DELETE", make_request=_mk_deregister)
+Client._register('get_profile', method='GET', url='users/me')
+Client._register('get_streams', method='GET', url='streams', make_request=_kwargs_to_dict)
+Client._register('get_members', method='GET', url='users')
+Client._register('list_subscriptions', method='GET', url='users/me/subscriptions')
+Client._register('add_subscriptions', url='users/me/subscriptions', make_request=_mk_subs)
+Client._register('remove_subscriptions', method='PATCH', url='users/me/subscriptions', make_request=_mk_rm_subs)
+Client._register('get_subscribers', method='GET',
+                 computed_url=lambda request: 'streams/%s/members' % (urllib.parse.quote(request['stream'], safe=''),),
+                 make_request=_kwargs_to_dict)
+Client._register('render_message', method='GET', url='messages/render')
+Client._register('create_user', method='POST', url='users')
--- a/static/assets/README
+++ b/static/assets/README
--- a/static/assets/default-default-avatar.svg
+++ b/static/assets/default-default-avatar.svg
--- a/static/assets/favicon/.gitignore
+++ b/static/assets/favicon/.gitignore
--- a/assets/favicon/generate
+++ b/assets/favicon/generate
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+import xml.etree.ElementTree as ET
+import subprocess
+from six.moves import range
+
+# Generates the favicon images containing unread message counts.
+
+# Open the SVG and find the number text elements using XPath
+tree  = ET.parse('orig.svg')
+elems = [tree.getroot().findall(
+    ".//*[@id='%s']/{http://www.w3.org/2000/svg}tspan" % (name,))[0]
+     for name in ('number_back', 'number_front')]
+
+for i in range(1, 100):
+    # Prepare a modified SVG
+    s = '%2d' % (i,)
+    for e in elems:
+        e.text = s
+    with open('tmp.svg', 'w') as out:
+        tree.write(out)
+
+    # Convert to PNG
+    subprocess.check_call(['inkscape', '--without-gui', '--export-area-page',
+        '--export-png=../../static/images/favicon/favicon-%d.png' % (i,),
+        'tmp.svg'])
--- a/static/assets/favicon/infinite.svg
+++ b/static/assets/favicon/infinite.svg
--- a/static/assets/favicon/orig.svg
+++ b/static/assets/favicon/orig.svg
--- a/static/assets/zulip-emoji/NOTICE
+++ b/static/assets/zulip-emoji/NOTICE
--- a/static/assets/zulip-emoji/README
+++ b/static/assets/zulip-emoji/README
--- a/static/assets/zulip-emoji/zulip.png
+++ b/static/assets/zulip-emoji/zulip.png
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tim Abbott	a063dd3b26	Update changelog for Zulip 1.4.3 release.	2017-01-29 15:09:02 -08:00
Tim Abbott	1cdd451d70	streams: Fix autosubscribe security bug (CVE-2017-0881). A bug in Zulip's implementation of the "stream exists" endpoint meant that any user of a Zulip server could subscribe to an invite-only stream without needing to be invited by using the "autosubscribe" argument. Thanks to Rafid Aslam for discovering this issue.	2017-01-29 15:09:02 -08:00
Tim Abbott	8cc7642cdd	Update changelog for Zulip 1.4.1 and 1.4.2 releases.	2016-09-27 20:09:20 -07:00
Tim Abbott	6883c916af	Upgrade Django to 1.8.15 with Zulip patches.	2016-09-27 20:09:20 -07:00
Tim Abbott	978a568c0f	puppet: Fix buggy logrotate configuration.	2016-09-27 20:01:17 -07:00
Steve Howell	f6975f9334	Upgrade: revert change to default LOCAL_UPLOADS_DIR in prod settings. The main purpose of the "var" convention is to make it easy to write stuff inside of our git repo when running a dev instance, and then "var" gets excluded from checkins. For production, that's not as much of a concern. For upgrades we don't want to be changing the directory around and confusing matters, especially with the extra moving part of nginx configs (which have their own issues in terms of being overwritten by accident when admins go to S3).	2016-09-02 12:30:29 -07:00
Christie Koehler	0120ff5612	upgrade: Create prod_settings symlink in step 2 if it doesn't exist. Between releases 1.3.13 and 1.4.0, local_settings.py was renamed to prod_settings.py. The upgrade scripts were adjusted to reflect this name change. But because the first part of the upgrade script is run with the currently installed version's code, the symlink to /etc/zulip/settings.py is created with the old name. This was causing upgrade-zulip-stage-2 to fail. Now upgrade-zulip-stage-2 creates the symlink at zproject/prod_settings.py if it doesn't already exist. Fixes #1731.	2016-09-01 21:17:11 -07:00