Disable use_socket manually for CUSTOMER13

This was causing tracebacks during message sending (imported from commit 9dde67f07934fe7a5aef7036340eebb7c6f6a97a)
2025-10-31 03:53:50 +00:00 · 2014-02-13 22:53:47 -05:00
4955 changed files with 199473 additions and 542435 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,161 +0,0 @@
 # See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
 #   high-level documentation on our CircleCI setup.
 # See CircleCI upstream's docs on this config format:
 #   https://circleci.com/docs/2.0/language-python/
 #
 version: 2
 aliases:
  - &create_cache_directories
    run:
      name: create cache directories
      command: |
          dirs=(/srv/zulip-{npm,venv}-cache)
          sudo mkdir -p "${dirs[@]}"
          sudo chown -R circleci "${dirs[@]}"
  - &restore_cache_package_json
    restore_cache:
      keys:
      - v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
  - &restore_cache_requirements
    restore_cache:
      keys:
      - v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
  - &install_dependencies
    run:
      name: install dependencies
      command: |
        sudo apt-get update
        # Install moreutils so we can use `ts` and `mispipe` in the following.
        sudo apt-get install -y moreutils
        # CircleCI sets the following in Git config at clone time:
        #   url.ssh://git@github.com.insteadOf https://github.com
        # This breaks the Git clones in the NVM `install.sh` we run
        # in `install-node`.
        # TODO: figure out why that breaks, and whether we want it.
        #   (Is it an optimization?)
        rm -f /home/circleci/.gitconfig
        # This is the main setup job for the test suite
        mispipe "tools/ci/setup-backend" ts
        # Cleaning caches is mostly unnecessary in Circle, because
        # most builds don't get to write to the cache.
        # mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
  - &save_cache_package_json
    save_cache:
      paths:
        - /srv/zulip-npm-cache
      key: v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
  - &save_cache_requirements
    save_cache:
      paths:
        - /srv/zulip-venv-cache
      key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
    # TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
  - &run_backend_tests
    run:
      name: run backend tests
      command: |
        . /srv/zulip-py3-venv/bin/activate
        mispipe ./tools/ci/backend ts
  - &run_frontend_tests
    run:
      name: run frontend tests
      command: |
        . /srv/zulip-py3-venv/bin/activate
        mispipe ./tools/ci/frontend ts
  - &upload_coverage_report
    run:
     name: upload coverage report
     command: |
       . /srv/zulip-py3-venv/bin/activate
       pip install codecov && codecov \
         || echo "Error in uploading coverage reports to codecov.io."
 jobs:
  "trusty-python-3.4":
    docker:
      # This is built from tools/circleci/images/trusty/Dockerfile .
      - image: gregprice/circleci:trusty-python-5.test
    working_directory: ~/zulip
    steps:
      - checkout
      - *create_cache_directories
      - *restore_cache_package_json
      - *restore_cache_requirements
      - *install_dependencies
      - *save_cache_package_json
      - *save_cache_requirements
      - *run_backend_tests
      - *run_frontend_tests
      - *upload_coverage_report
      # - store_artifacts:  # TODO
      #     path: var/casper/
      #     # also /tmp/zulip-test-event-log/
      #     destination: test-reports
  "xenial-python-3.5":
    docker:
      # This is built from tools/circleci/images/xenial/Dockerfile .
      - image: gregprice/circleci:xenial-python-4.test
    working_directory: ~/zulip
    steps:
      - checkout
      - *create_cache_directories
      - *restore_cache_package_json
      - *restore_cache_requirements
      - *install_dependencies
      - *save_cache_package_json
      - *save_cache_requirements
      - *run_backend_tests
      - *upload_coverage_report
  "bionic-python-3.6":
    docker:
      # This is built from tools/circleci/images/bionic/Dockerfile .
      - image: gregprice/circleci:bionic-python-1.test
    working_directory: ~/zulip
    steps:
      - checkout
      - *create_cache_directories
      - run:
          name: do Bionic hack
          command: |
              # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
              # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
              redis-server --daemonize yes
      - *restore_cache_package_json
      - *restore_cache_requirements
      - *install_dependencies
      - *save_cache_package_json
      - *save_cache_requirements
      - *run_backend_tests
      - *upload_coverage_report
 workflows:
  version: 2
  build:
    jobs:
      - "trusty-python-3.4"
      - "xenial-python-3.5"
      - "bionic-python-3.6"
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,10 +0,0 @@
 comment: off
 coverage:
  status:
    project:
      default:
        target: auto
        threshold: 0.50
        base: auto
    patch: off
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,25 +0,0 @@
 root = true
 [*]
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
 [*.{sh,py,pyi,js,json,yml,xml,css,md,markdown,handlebars,html}]
 indent_style = space
 indent_size = 4
 [*.{py}]
 max_line_length = 110
 [*.{js}]
 max_line_length = 120
 [*.{svg,rb,pp,pl}]
 indent_style = space
 indent_size = 2
 [*.{cfg}]
 indent_style = space
 indent_size = 8
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,2 +0,0 @@
 static/js/blueslip.js
 static/webpack-bundles
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,411 +0,0 @@
 {
    "env": {
        "node": true,
        "es6": true
    },
    "parserOptions": {
        "sourceType": "module"
    },
    "globals": {
        "$": false,
        "ClipboardJS": false,
        "Dict": false,
        "FetchStatus": false,
        "Filter": false,
        "Handlebars": false,
        "LightboxCanvas": false,
        "MessageListData": false,
        "MessageListView": false,
        "PerfectScrollbar": false,
        "Plotly": false,
        "SockJS": false,
        "Socket": false,
        "Sortable": false,
        "WinChan": false,
        "XDate": false,
        "_": false,
        "activity": false,
        "admin": false,
        "alert_words": false,
        "alert_words_ui": false,
        "attachments_ui": false,
        "avatar": false,
        "billing": false,
        "blueslip": false,
        "bot_data": false,
        "bridge": false,
        "buddy_data": false,
        "buddy_list": false,
        "channel": false,
        "click_handlers": false,
        "color_data": false,
        "colorspace": false,
        "common": false,
        "components": false,
        "compose": false,
        "compose_actions": false,
        "compose_fade": false,
        "compose_pm_pill": false,
        "compose_state": false,
        "compose_ui": false,
        "composebox_typeahead": false,
        "condense": false,
        "confirm_dialog": false,
        "copy_and_paste": false,
        "csrf_token": false,
        "current_msg_list": true,
        "drafts": false,
        "echo": false,
        "emoji": false,
        "emoji_codes": false,
        "emoji_picker": false,
        "favicon": false,
        "feature_flags": false,
        "feedback_widget": false,
        "fenced_code": false,
        "flatpickr": false,
        "floating_recipient_bar": false,
        "gear_menu": false,
        "hash_util": false,
        "hashchange": false,
        "helpers": false,
        "home_msg_list": false,
        "hotspots": false,
        "i18n": false,
        "info_overlay": false,
        "input_pill": false,
        "invite": false,
        "jQuery": false,
        "katex": false,
        "keydown_util": false,
        "lightbox": false,
        "list_cursor": false,
        "list_render": false,
        "list_util": false,
        "loading": false,
        "localStorage": false,
        "local_message": false,
        "localstorage": false,
        "markdown": false,
        "marked": false,
        "md5": false,
        "message_edit": false,
        "message_events": false,
        "message_fetch": false,
        "message_flags": false,
        "message_list": false,
        "message_live_update": false,
        "message_scroll": false,
        "message_store": false,
        "message_util": false,
        "message_viewport": false,
        "moment": false,
        "muting": false,
        "muting_ui": false,
        "narrow": false,
        "narrow_state": false,
        "navigate": false,
        "night_mode": false,
        "notifications": false,
        "overlays": false,
        "padded_widget": false,
        "page_params": false,
        "panels": false,
        "people": false,
        "pm_conversations": false,
        "pm_list": false,
        "pointer": false,
        "popovers": false,
        "presence": false,
        "pygments_data": false,
        "reactions": false,
        "realm_icon": false,
        "realm_logo": false,
        "realm_night_logo": false,
        "recent_senders": false,
        "reload": false,
        "reload_state": false,
        "reminder": false,
        "resize": false,
        "rows": false,
        "rtl": false,
        "run_test": false,
        "schema": false,
        "scroll_bar": false,
        "scroll_util": false,
        "search": false,
        "search_pill": false,
        "search_pill_widget": false,
        "search_suggestion": false,
        "search_util": false,
        "sent_messages": false,
        "server_events": false,
        "server_events_dispatch": false,
        "settings": false,
        "settings_account": false,
        "settings_bots": false,
        "settings_display": false,
        "settings_emoji": false,
        "settings_linkifiers": false,
        "settings_invites": false,
        "settings_muting": false,
        "settings_notifications": false,
        "settings_org": false,
        "settings_panel_menu": false,
        "settings_profile_fields": false,
        "settings_sections": false,
        "settings_streams": false,
        "settings_toggle": false,
        "settings_ui": false,
        "settings_user_groups": false,
        "settings_users": false,
        "starred_messages": false,
        "stream_color": false,
        "stream_create": false,
        "stream_data": false,
        "stream_edit": false,
        "stream_events": false,
        "stream_list": false,
        "stream_muting": false,
        "stream_popover": false,
        "stream_sort": false,
        "StripeCheckout": false,
        "submessage": false,
        "subs": false,
        "tab_bar": false,
        "templates": false,
        "tictactoe_widget": false,
        "timerender": false,
        "toMarkdown": false,
        "todo_widget": false,
        "top_left_corner": false,
        "topic_data": false,
        "topic_generator": false,
        "topic_list": false,
        "topic_zoom": false,
        "transmit": false,
        "tutorial": false,
        "typeahead_helper": false,
        "typing": false,
        "typing_data": false,
        "typing_events": false,
        "typing_status": false,
        "ui": false,
        "ui_init": false,
        "ui_report": false,
        "ui_util": false,
        "unread": false,
        "unread_ops": false,
        "unread_ui": false,
        "upgrade": false,
        "upload": false,
        "upload_widget": false,
        "user_events": false,
        "user_groups": false,
        "user_pill": false,
        "user_search": false,
        "user_status": false,
        "user_status_ui": false,
        "util": false,
        "poll_widget": false,
        "widgetize": false,
        "zcommand": false,
        "zform": false,
        "zxcvbn": false
    },
    "plugins": [
        "eslint-plugin-empty-returns"
    ],
    "rules": {
        "array-callback-return": "error",
        "array-bracket-spacing": "error",
        "arrow-spacing": [ "error", { "before": true, "after": true } ],
        "block-scoped-var": 2,
        "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
        "camelcase": 0,
        "comma-dangle": [ "error",
            {
                "arrays": "always-multiline",
                "objects": "always-multiline",
                "imports": "always-multiline",
                "exports": "always-multiline",
                "functions": "never"
            }
        ],
        "comma-spacing": [ "error",
            {
                "before": false,
                "after": true
            }
        ],
        "complexity": [ 0, 4 ],
        "curly": 2,
        "dot-notation": [ "error", { "allowKeywords": true } ],
        "empty-returns/main": "error",
        "eol-last": [ "error", "always" ],
        "eqeqeq": 2,
        "func-style": [ "off", "expression" ],
        "guard-for-in": 2,
        "indent": ["error", 4, {
            "ArrayExpression": "first",
            "outerIIFEBody": 0,
            "ObjectExpression": "first",
            "SwitchCase": 0,
            "CallExpression": {"arguments": "first"},
            "FunctionExpression": {"parameters": "first"},
            "FunctionDeclaration": {"parameters": "first"}
        }],
        "key-spacing": [ "error",
            {
                "beforeColon": false,
                "afterColon": true
            }
        ],
        "keyword-spacing": [ "error",
            {
                "before": true,
                "after": true,
                "overrides": {
                    "return": { "after": true },
                    "throw": { "after": true },
                    "case": { "after": true }
                }
            }
        ],
        "max-depth": [ 0, 4 ],
        "max-len": [ "error", 100, 2,
            {
                "ignoreUrls": true,
                "ignoreComments": false,
                "ignoreRegExpLiterals": true,
                "ignoreStrings": true,
                "ignoreTemplateLiterals": true
            }
        ],
        "max-params": [ 0, 3 ],
        "max-statements": [ 0, 10 ],
        "new-cap": [ "error",
            {
                "newIsCap": true,
                "capIsNew": false
            }
        ],
        "new-parens": 2,
        "newline-per-chained-call": 0,
        "no-alert": 2,
        "no-array-constructor": "error",
        "no-bitwise": 2,
        "no-caller": 2,
        "no-case-declarations": "error",
        "no-catch-shadow": 2,
        "no-console": 0,
        "no-const-assign": "error",
        "no-control-regex": 2,
        "no-debugger": 2,
        "no-delete-var": 2,
        "no-div-regex": 2,
        "no-dupe-class-members": "error",
        "no-dupe-keys": 2,
        "no-duplicate-imports": "error",
        "no-else-return": 2,
        "no-empty": 2,
        "no-empty-character-class": 2,
        "no-eq-null": 2,
        "no-eval": 2,
        "no-ex-assign": 2,
        "no-extra-parens": ["error", "all"],
        "no-extra-semi": 2,
        "no-fallthrough": 2,
        "no-floating-decimal": 2,
        "no-func-assign": 2,
        "no-implied-eval": 2,
        "no-iterator": "error",
        "no-label-var": 2,
        "no-labels": 2,
        "no-loop-func": 2,
        "no-mixed-requires": [ 0, false ],
        "no-multi-str": 2,
        "no-native-reassign": 2,
        "no-nested-ternary": 0,
        "no-new-func": "error",
        "no-new-object": 2,
        "no-new-wrappers": 2,
        "no-obj-calls": 2,
        "no-octal": 2,
        "no-octal-escape": 2,
        "no-param-reassign": 0,
        "no-plusplus": 2,
        "no-proto": 2,
        "no-redeclare": 2,
        "no-regex-spaces": 2,
        "no-restricted-syntax": 0,
        "no-return-assign": 2,
        "no-script-url": 2,
        "no-self-compare": 2,
        "no-shadow": 0,
        "no-sync": 2,
        "no-ternary": 0,
        "no-trailing-spaces": 2,
        "no-undef": "error",
        "no-undef-init": 2,
        "no-underscore-dangle": 0,
        "no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
        "no-unreachable": 2,
        "no-unused-expressions": 2,
        "no-unused-vars": [ "error",
            {
                "vars": "local",
                "args": "after-used",
                "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
            }
        ],
        "no-use-before-define": 2,
        "no-useless-constructor": "error",
        // The Zulip codebase complies partially with the "no-useless-escape"
        // rule; only regex expressions haven't been updated yet.
        // Updated regex expressions are currently being tested in casper
        // files and will decide about a potential future enforcement of this rule.
        "no-useless-escape": 0,
        "space-unary-ops": 2,
        "no-whitespace-before-property": 2,
        "no-with": 2,
        "one-var": [ "error", "never" ],
        "padded-blocks": 0,
        "prefer-const": [ "error",
            {
                "destructuring": "any",
                "ignoreReadBeforeAssign": true
            }
        ],
        "quote-props": [ "error", "as-needed",
            {
                "keywords": false,
                "unnecessary": true,
                "numbers": false
            }
        ],
        "quotes": [ 0, "single" ],
        "radix": 2,
        "semi": 2,
        "semi-spacing": [2, {"before": false, "after": true}],
        "space-before-blocks": 2,
        "space-before-function-paren": [ "error",
            {
                "anonymous": "always",
                "named": "never",
                "asyncArrow": "always"
            }
        ],
        "space-in-parens": 2,
        "space-infix-ops": 2,
        "spaced-comment": 0,
        "strict": 0,
        "template-curly-spacing": "error",
        "unnecessary-strict": 0,
        "use-isnan": 2,
        "valid-typeof": [ "error", { "requireStringLiterals": true } ],
        "wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
        "wrap-regex": 0,
        "yoda": 2
    }
 }
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,14 +1,20 @@
-*   text=auto eol=lf
+.gitignore export-ignore
-*.gif binary
+.gitattributes export-ignore
-*.jpg binary
+/analytics export-ignore
-*.jpeg binary
+/assets export-ignore
-*.eot binary
+/bots export-ignore
-*.woff binary
+/corporate export-ignore
-*.woff2 binary
+/static export-ignore
-*.svg binary
+/tools export-ignore
-*.ttf binary
+/zilencer export-ignore
-*.png binary
+/templates/analytics export-ignore
-*.otf binary
+/templates/corporate export-ignore
-*.tif binary
+/templates/zilencer export-ignore
-*.ogg binary
+/puppet/zulip_internal export-ignore
-yarn.lock binary
+/zproject/local_settings.py export-ignore
 /zproject/test_settings.py export-ignore
 /zerver/fixtures export-ignore
 /zerver/tests.py export-ignore
 /zerver/tests export-ignore
 /node_modules export-ignore
 /humbug export-ignore
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,14 +0,0 @@
 <!-- What's this PR for?  (Just a link to an issue is fine.) -->
 **Testing Plan:** <!-- How have you tested? -->
 **GIFs or Screenshots:** <!-- If a UI change.  See:
  https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
  -->
 <!-- Also be sure to make clear, coherent commits:
  https://zulip.readthedocs.io/en/latest/contributing/version-control.html
  -->
--- a/.gitignore
+++ b/.gitignore
@@ -1,75 +1,28 @@
 # Quick format and style primer:
 #
 #  * If a pattern is meant only for a specific location, it should have a
 #    leading slash, like `/staticfiles.json`.
 #    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
 #      will do, but this makes a confusing pattern.  Adding a leading slash
 #      is clearer.
 #
 #  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
 #    match in any subdirectory.
 #
 #  * Subdirectories with several internal things to ignore get their own
 #    `.gitignore` files.
 #
 #  * Comments must be on their own line.  (Otherwise they don't work.)
 #
 # See `git help ignore` for details on the format.
 ## Config files for the dev environment
 /zproject/dev-secrets.conf
 /tools/conf.ini
 /tools/custom_provision
 /tools/droplets/conf.ini
 ## Byproducts of setting up and using the dev environment
 *.pyc
 package-lock.json
 /.vagrant
 /var
 /.dmypy.json
 # Dockerfiles generated for CircleCI
 /tools/circleci/images
 # Static build
 *.mo
 npm-debug.log
 /node_modules
 /prod-static
 /staticfiles.json
 /webpack-stats-production.json
 /yarn-error.log
 # Test / analysis tools
 .coverage
 ## Files (or really symlinks) created in a prod deployment
 /zproject/prod_settings.py
 /zulip-current-venv
 /zulip-py3-venv
 ## Files left by various editors and local environments
 # (Ideally these should be in everyone's respective personal gitignore files.)
 *~
 /all_messages_log.*
 /event_log/*
 /server.log
 /update-prod-static.log
 /prod-static
 /errors/*
 *.sw[po]
-.idea
+.DS_Store
 event_queues.pickle
 stats/
 zerver/fixtures/available-migrations
 zerver/fixtures/migration-status
 zerver/fixtures/test_data1.json
 .kdev4
 zulip.kdev4
 memcached_prefix
 coverage/
 /queue_error
 /workers.log
 .test-js-with-node.html
 digest.log
 errors.log
 manage.log
 .kateproject.d/
 .kateproject
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
 .vscode/
 *.DS_Store
 # .cache/ is generated by VSCode's test runner
 .cache/
 .eslintcache
 ## Miscellaneous
 # (Ideally this section is empty.)
 zthumbor/thumbor_local_settings.py
 .transifexrc
--- a/.gitlint
+++ b/.gitlint
@@ -1,13 +0,0 @@
 [general]
 ignore=title-trailing-punctuation, body-min-length, body-is-missing, title-imperative-mood
 extra-path=tools/lib/gitlint-rules.py
 [title-match-regex-allow-exception]
 regex=^(.+:\ )?[A-Z].+\.$
 [title-max-length]
 line-length=76
 [body-max-line-length]
 line-length=76
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,10 +0,0 @@
 [settings]
 line_length = 79
 multi_line_output = 2
 balanced_wrapping = true
 known_third_party = django, ujson, sqlalchemy
 known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
 sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
 lines_after_imports = 1
 # See the comment related to ioloop_logging for why this is skipped.
 skip = zerver/management/commands/runtornado.py
--- a/.stylelintrc
+++ b/.stylelintrc
@@ -1,67 +0,0 @@
 {
    "rules": {
        # Stylistic rules for CSS.
        "function-comma-space-after": "always",
        "function-comma-space-before": "never",
        "function-max-empty-lines": 0,
        "function-whitespace-after": "always",
        "value-keyword-case": "lower",
        "value-list-comma-newline-after": "always-multi-line",
        "value-list-comma-space-after": "always-single-line",
        "value-list-comma-space-before": "never",
        "value-list-max-empty-lines": 0,
        "unit-case": "lower",
        "property-case": "lower",
        "color-hex-case": "lower",
        "declaration-bang-space-before": "always",
        "declaration-colon-newline-after": "always-multi-line",
        "declaration-colon-space-after": "always-single-line",
        "declaration-colon-space-before": "never",
        "declaration-block-semicolon-newline-after": "always",
        "declaration-block-semicolon-space-before": "never",
        "declaration-block-trailing-semicolon": "always",
        "block-closing-brace-empty-line-before": "never",
        "block-closing-brace-newline-after": "always",
        "block-closing-brace-newline-before": "always",
        "block-opening-brace-newline-after": "always",
        "block-opening-brace-space-before": "always",
        "selector-attribute-brackets-space-inside": "never",
        "selector-attribute-operator-space-after": "never",
        "selector-attribute-operator-space-before": "never",
        "selector-combinator-space-after": "always",
        "selector-combinator-space-before": "always",
        "selector-descendant-combinator-no-non-space": true,
        "selector-pseudo-class-parentheses-space-inside": "never",
        "selector-pseudo-element-case": "lower",
        "selector-pseudo-element-colon-notation": "double",
        "selector-type-case": "lower",
        "selector-list-comma-newline-after": "always",
        "selector-list-comma-space-before": "never",
        "media-feature-colon-space-after": "always",
        "media-feature-colon-space-before": "never",
        "media-feature-name-case": "lower",
        "media-feature-parentheses-space-inside": "never",
        "media-feature-range-operator-space-after": "always",
        "media-feature-range-operator-space-before": "always",
        "media-query-list-comma-newline-after": "always",
        "media-query-list-comma-space-before": "never",
        "at-rule-name-case": "lower",
        "at-rule-name-space-after": "always",
        "at-rule-semicolon-newline-after": "always",
        "at-rule-semicolon-space-before": "never",
        "comment-whitespace-inside": "always",
        "indentation": 4,
        # Limit language features
        "color-no-hex": true,
        "color-named": "never",
    }
 }
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,67 +0,0 @@
 # See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
 # high-level documentation on our Travis CI setup.
 dist: trusty
 group: deprecated-2017Q4
 install:
  # Disable sometimes-broken sources.list in Travis base images
  - sudo rm -vf /etc/apt/sources.list.d/*
  - sudo apt-get update
  # Disable Travis CI's built-in NVM installation
  - mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
  # Install codecov, the library for the code coverage reporting tool we use
  # With a retry to minimize impact of transient networking errors.
  - mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
  # This is the main setup job for the test suite
  - mispipe "tools/ci/setup-$TEST_SUITE" ts
  # Clean any caches that are not in use to avoid our cache
  # becoming huge.
  - mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
 script:
  # We unset GEM_PATH here as a hack to work around Travis CI having
  # broken running their system puppet with Ruby.  See
  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
  - unset GEM_PATH
  - mispipe "./tools/ci/$TEST_SUITE" ts
 cache:
  yarn: true
  apt: false
  directories:
    - $HOME/zulip-venv-cache
    - $HOME/zulip-npm-cache
    - $HOME/zulip-emoji-cache
    - $HOME/node
    - $HOME/misc
 env:
  global:
    - BOTO_CONFIG=/nonexistent
 language: python
 # Our test suites generally run on Python 3.4, the version in
 # Ubuntu 14.04 trusty, which is the oldest OS release we support.
 matrix:
  include:
    # Travis will actually run the jobs in the order they're listed here;
    # that doesn't seem to be documented, but it's what we see empirically.
    # We only get 4 jobs running at a time, so we try to make the first few
    # the most likely to break.
    - python: "3.4"
      env: TEST_SUITE=production
    # Other suites moved to CircleCI -- see .circleci/.
 sudo: required
 addons:
  artifacts:
    paths:
      # Casper debugging data (screenshots, etc.) is super useful for
      # debugging test flakes.
      - $(ls var/casper/* | tr "\n" ":")
      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
  postgresql: "9.3"
  apt:
    packages:
      - moreutils
 after_success:
  - codecov
--- a/.tx/config
+++ b/.tx/config
@@ -1,33 +0,0 @@
 [main]
 host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 [zulip.djangopo]
 source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 file_filter = static/locale/<lang>/LC_MESSAGES/django.po
 [zulip.translationsjson]
 source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
 [zulip.mobile]
 source_file = static/locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/mobile.json
 [zulip-test.djangopo]
 source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 file_filter = static/locale/<lang>/LC_MESSAGES/django.po
 [zulip-test.translationsjson]
 source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,105 +0,0 @@
 # Zulip Code of Conduct
 Like the technical community as a whole, the Zulip team and community is
 made up of a mixture of professionals and volunteers from all over the
 world, working on every aspect of the mission, including mentorship,
 teaching, and connecting people.
 Diversity is one of our huge strengths, but it can also lead to
 communication issues and unhappiness. To that end, we have a few ground
 rules that we ask people to adhere to. This code applies equally to
 founders, mentors, and those seeking help and guidance.
 This isn't an exhaustive list of things that you can't do. Rather, take it
 in the spirit in which it's intended --- a guide to make it easier to enrich
 all of us and the technical communities in which we participate.
 ## Expected Behavior
 The following behaviors are expected and requested of all community members:
 * Participate. In doing so, you contribute to the health and longevity of
  the community.
 * Exercise consideration and respect in your speech and actions.
 * Attempt collaboration before conflict. Assume good faith.
 * Refrain from demeaning, discriminatory, or harassing behavior and speech.
 * Take action or alert community leaders if you notice a dangerous
  situation, someone in distress, or violations of this code, even if they
  seem inconsequential.
 * Community event venues may be shared with members of the public; be
  respectful to all patrons of these locations.
 ## Unacceptable Behavior
 The following behaviors are considered harassment and are unacceptable
 within the Zulip community:
 * Jokes or derogatory language that singles out members of any race,
  ethnicity, culture, national origin, color, immigration status, social and
  economic class, educational level, language proficiency, sex, sexual
  orientation, gender identity and expression, age, size, family status,
  political belief, religion, and mental and physical ability.
 * Violence, threats of violence, or violent language directed against
  another person.
 * Disseminating or threatening to disseminate another person's personal
  information.
 * Personal insults of any sort.
 * Posting or displaying sexually explicit or violent material.
 * Inappropriate photography or recording.
 * Deliberate intimidation, stalking, or following (online or in person).
 * Unwelcome sexual attention. This includes sexualized comments or jokes,
  inappropriate touching or groping, and unwelcomed sexual advances.
 * Sustained disruption of community events, including talks and
  presentations.
 * Advocating for, or encouraging, any of the behaviors above.
 ## Reporting and Enforcement
 Harassment and other code of conduct violations reduce the value of the
 community for everyone. If someone makes you or anyone else feel unsafe or
 unwelcome, please report it to the community organizers at
 zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
 report either personally or anonymously.
 If a community member engages in unacceptable behavior, the community
 organizers may take any action they deem appropriate, up to and including a
 temporary ban or permanent expulsion from the community without warning (and
 without refund in the case of a paid event).
 If someone outside the development community (e.g. a user of the Zulip
 software) engages in unacceptable behavior that affects someone in the
 community, we still want to know. Even if we don't have direct control over
 the violator, the community organizers can still support the people
 affected, reduce the chance of a similar violation in the future, and take
 any direct action we can.
 The nature of reporting means it can only help after the fact. If you see
 something you can do while a violation is happening, do it. A lot of the
 harms of harassment and other violations can be mitigated by the victim
 knowing that the other people present are on their side.
 All reports will be kept confidential. In some cases we may determine that a
 public statement will need to be made. In such cases, the identities of all
 victims and reporters will remain confidential unless those individuals
 instruct us otherwise.
 ## Scope
 We expect all community participants (contributors, paid or otherwise,
 sponsors, and other guests) to abide by this Code of Conduct in all
 community venues, online and in-person, as well as in all private
 communications pertaining to community business.
 This Code of Conduct and its related procedures also applies to unacceptable
 behavior occurring outside the scope of community activities when such
 behavior has the potential to adversely affect the safety and well-being of
 community members.
 ## License and Attribution
 This Code of Conduct is adapted from the
 [Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
 [Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
 under a
 [Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
 license.
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,342 +0,0 @@
 # Contributing to Zulip
 Welcome to the Zulip community!
 ## Community
 The
 [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
 is the primary communication forum for the Zulip community. It is a good
 place to start whether you have a question, are a new contributor, are a new
 user, or anything else. Make sure to read the
 [community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
 before posting. The Zulip community is also governed by a
 [code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
 You can subscribe to zulip-devel@googlegroups.com for a lower traffic (~1
 email/month) way to hear about things like mentorship opportunities with Google
 Code-in, in-person sprints at conferences, and other opportunities to
 contribute.
 ## Ways to contribute
 To make a code or documentation contribution, read our
 [step-by-step guide](#your-first-codebase-contribution) to getting
 started with the Zulip codebase. A small sample of the type of work that
 needs doing:
 * Bug squashing and feature development on our Python/Django
  [backend](https://github.com/zulip/zulip), web
  [frontend](https://github.com/zulip/zulip), React Native
  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
  [desktop app](https://github.com/zulip/zulip-electron).
 * Building out our
  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
 * [Writing an integration](https://zulipchat.com/api/integrations-overview).
 * Improving our [user](https://zulipchat.com/help/) or
  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
 * [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
  and manually testing pull requests.
 **Non-code contributions**: Some of the most valuable ways to contribute
 don't require touching the codebase at all. We list a few of them below:
 * [Reporting issues](#reporting-issues), including both feature requests and
  bug reports.
 * [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
 * [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
  Zulip.
 * [Outreach](#zulip-outreach): Star us on GitHub, upvote us
  on product comparison sites, or write for [the Zulip blog](http://blog.zulip.org/).
 ## Your first (codebase) contribution
 This section has a step by step guide to starting as a Zulip codebase
 contributor. It's long, but don't worry about doing all the steps perfectly;
 no one gets it right the first time, and there are a lot of people available
 to help.
 * First, make an account on the
  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
  paying special attention to the community norms. If you'd like, introduce
  yourself in
  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
  your name as the topic. Bonus: tell us about your first impressions of
  Zulip, and anything that felt confusing/broken as you started using the
  product.
 * Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
 * [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
  getting help in
  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
  if you run into any troubles.
 * Read the
  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
  and do the Git tutorial (coming soon) if you are unfamiliar with
  Git, getting help in
  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
  you run into any troubles.  Be sure to check out the
  [extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
 * Sign the
  [Dropbox Contributor License Agreement](https://opensource.dropbox.com/cla/).
 ### Picking an issue
 Now, you're ready to pick your first issue! There are hundreds of open issues
 in the main codebase alone. This section will help you find an issue to work
 on.
 * If you're interested in
  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
  [desktop](https://github.com/zulip/zulip-electron/issues?q=is%3Aopen+is%3Aissue),
  or
  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
  development, check the respective links for open issues, or post in
  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
 * For the main server and web repository, start by looking through issues
  with the label
  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A"good+first+issue").
  These are smaller projects particularly suitable for a first contribution.
 * We also partition all of our issues in the main repo into areas like
  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
  through our [list of labels](https://github.com/zulip/zulip/labels), and
  click on some of the `area:` labels to see all the issues related to your
  areas of interest.
 * If the lists of issues are overwhelming, post in
  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
  bit about your background and interests, and we'll help you out. The most
  important thing to say is whether you're looking for a backend (Python),
  frontend (JavaScript), mobile (React Native), desktop (Electron),
  documentation (English) or visual design (JavaScript + CSS) issue, and a
  bit about your programming experience and available time.
 We also welcome suggestions of features that you feel would be valuable or
 changes that you feel would make Zulip a better open source project. If you
 have a new feature you'd like to add, we recommend you start by posting in
 [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
 feature idea and the problem that you're hoping to solve.
 Other notes:
 * For a first pull request, it's better to aim for a smaller contribution
  than a bigger one. Many first contributions have fewer than 10 lines of
  changes (not counting changes to tests).
 * The full list of issues looking for a contributor can be found with the
  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
  and
  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
  labels.
 * For most new contributors, there's a lot to learn while making your first
  pull request. It's OK if it takes you a while; that's normal! You'll be
  able to work a lot faster as you build experience.
 ### Working on an issue
 To work on an issue, claim it by adding a comment with `@zulipbot claim` to
 the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
 workflow bot; it will assign you to the issue and label the issue as "in
 progress". Some additional notes:
 * You can only claim issues with the
  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
  or
  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
  labels. Zulipbot will give you an error if you try to claim an issue
  without one of those labels.
 * You're encouraged to ask questions on how to best implement or debug your
  changes -- the Zulip maintainers are excited to answer questions to help
  you stay unblocked and working efficiently. You can ask questions on
  chat.zulip.org, or on the GitHub issue or pull request.
 * We encourage early pull requests for work in progress. Prefix the title of
  work in progress pull requests with `[WIP]`, and remove the prefix when
  you think it might be mergeable and want it to be reviewed.
 * After updating a PR, add a comment to the GitHub thread mentioning that it
  is ready for another review. GitHub only notifies maintainers of the
  changes when you post a comment, so if you don't, your PR will likely be
  neglected by accident!
 ### And beyond
 A great place to look for a second issue is to look for issues with the same
 `area:` label as the last issue you resolved. You'll be able to reuse the
 work you did learning how that part of the codebase works. Also, the path to
 becoming a core developer often involves taking ownership of one of these area
 labels.
 ## What makes a great Zulip contributor?
 Zulip runs a lot of [internship programs](#internship-programs), so we have
 a lot of experience with new contributors. In our experience, these are the
 best predictors of success:
 * Posting good questions. This generally means explaining your current
  understanding, saying what you've done or tried so far, and including
  tracebacks or other error messages if appropriate.
 * Learning and practicing
  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
 * Submitting carefully tested code. This generally means checking your work
  through a combination of automated tests and manually clicking around the
  UI trying to find bugs in your work. See
  [things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
  for additional ideas.
 * Posting
  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
  for frontend changes.
 * Being responsive to feedback on pull requests. This means incorporating or
  responding to all suggested changes, and leaving a note if you won't be
  able to address things within a few days.
 * Being helpful and friendly on chat.zulip.org.
 These are also the main criteria we use to select interns for all of our
 internship programs.
 ## Reporting issues
 If you find an easily reproducible bug and/or are experienced in reporting
 bugs, feel free to just open an issue on the relevant project on GitHub.
 If you have a feature request or are not yet sure what the underlying bug
 is, the best place to post issues is
 [#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
 [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
 [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
 [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
 This allows us to interactively figure out what is going on, let you know if
 a similar issue has already been opened, and collect any other information
 we need. Choose a 2-4 word topic that describes the issue, explain the issue
 and how to reproduce it if known, your browser/OS if relevant, and a
 [screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
 if appropriate.
 **Reporting security issues**. Please do not report security issues
  publicly, including on public streams on chat.zulip.org. You can email
  zulip-security@googlegroups.com. We create a CVE for every security issue.
 ## User feedback
 Nearly every feature we develop starts with a user request. If you are part
 of a group that is either using or considering using Zulip, we would love to
 hear about your experience with the product. If you're not sure what to
 write, here are some questions we're always very curious to know the answer
 to:
 * Evaluation: What is the process by which your organization chose or will
  choose a group chat product?
 * Pros and cons: What are the pros and cons of Zulip for your organization,
  and the pros and cons of other products you are evaluating?
 * Features: What are the features that are most important for your
  organization? In the best case scenario, what would your chat solution do
  for you?
 * Onboarding: If you remember it, what was your impression during your first
  few minutes of using Zulip? What did you notice, and how did you feel? Was
  there anything that stood out to you as confusing, or broken, or great?
 * Organization: What does your organization do? How big is the organization?
  A link to your organization's website?
 ## Internship programs
 Zulip runs internship programs with
 [Outreachy](https://www.outreachy.org/),
 [Google Summer of Code (GSoC)](https://developers.google.com/open-source/gsoc/)
 [1], and the
 [MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram),
 and has in the past taken summer interns from Harvard, MIT, and
 Stanford.
 While each third-party program has its own rules and requirements, the
 Zulip community's approaches all of these programs with these ideas in
 mind:
 * We try to make the application process as valuable for the applicant as
  possible. Expect high quality code reviews, a supportive community, and
  publicly viewable patches you can link to from your resume, regardless of
  whether you are selected.
 * To apply, you'll have to submit at least one pull request to a Zulip
  repository.  Most students accepted to one of our programs have
  several merged pull requests (including at least one larger PR) by
  the time of the application deadline.
 * The main criteria we use is quality of your best contributions, and
  the bullets listed at
  [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
  Because we focus on evaluating your best work, it doesn't hurt your
  application to makes mistakes in your first few PRs as long as your
  work improves.
 Zulip also participates in
 [Google Code-In](https://developers.google.com/open-source/gci/). Our
 selection criteria for Finalists and Grand Prize Winners is the same as our
 selection criteria for interns above.
 Most of our interns end up sticking around the project long-term, and many
 quickly become core team members. We hope you apply!
 ### Google Summer of Code
 GSoC is by far the largest of our internship programs (we had 14 GSoC
 students in summer 2017).  While we don't control how many slots
 Google allocates to Zulip, we hope to mentor a similar number of
 students in 2018.
 If you're reading this well before the application deadline and want
 to make your application strong, we recommend getting involved in the
 community and fixing issues in Zulip now. Having good contributions
 and building a reputation for doing good work is best way to have a
 strong application.  About half of Zulip's GSoC students for Summer
 2017 had made significant contributions to the project by February
 2017, and about half had not.  Our
 [GSoC project ideas page][gsoc-guide] has lots more details on how
 Zulip does GSoC, as well as project ideas (though the project idea
 list is maintained only during the GSoC application period, so if
 you're looking at some other time of year, the project list is likely
 out-of-date).
 We also have in some past years run a Zulip Summer of Code (ZSoC)
 program for students who we didn't have enough slots to accept for
 GSoC but were able to find funding for.  Student expectations are the
 same as with GSoC, and it has no separate application process; your
 GSoC application is your ZSoC application.  If we'd like to select you
 for ZSoC, we'll contact you when the GSoC results are announced.
 [gsoc-guide]: https://zulip.readthedocs.io/en/latest/overview/gsoc-ideas.html
 [gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
 [1] Formally, [GSoC isn't an internship][gsoc-faq], but it is similar
 enough that we're treating it as such for the purposes of this
 documentation.
 ## Zulip Outreach
 **Upvoting Zulip**. Upvotes and reviews make a big difference in the public
 perception of projects like Zulip. We've collected a few sites below
 where we know Zulip has been discussed. Doing everything in the following
 list typically takes about 15 minutes.
 * Star us on GitHub. There are four main repositories:
  [server/web](https://github.com/zulip/zulip),
  [mobile](https://github.com/zulip/zulip-mobile),
  [desktop](https://github.com/zulip/zulip-electron), and
  [Python API](https://github.com/zulip/python-zulip-api).
 * [Follow us](https://twitter.com/zulip) on Twitter.
 For both of the following, you'll need to make an account on the site if you
 don't already have one.
 * [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
  AlternativeTo. We recommend upvoting a couple of other products you like
  as well, both to give back to their community, and since single-upvote
  accounts are generally given less weight. You can also
  [upvote Zulip](https://alternativeto.net/software/slack/) on their page
  for Slack.
 * [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
  it, and upvote the reasons why people like Zulip that you find most
  compelling. Again, we recommend adding a few other products that you like
  as well.
 We have a doc with more detailed instructions and a few other sites, if you
 have been using Zulip for a while and want to contribute more.
 **Blog posts**. Writing a blog post about your experiences with Zulip, or
 about a technical aspect of Zulip can be a great way to spread the word
 about Zulip.
 We also occasionally [publish](http://blog.zulip.org/) longer form
 articles related to Zulip. Our posts typically get tens of thousands
 of views, and we always have good ideas for blog posts that we can
 outline but don't have time to write. If you are an experienced writer
 or copyeditor, send us a portfolio; we'd love to talk!
--- a/17
+++ b/17
@@ -1,17 +0,0 @@
 FROM ubuntu:trusty
 EXPOSE 9991
 RUN apt-get update && apt-get install -y wget
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER zulip
 RUN ln -nsf /srv/zulip ~/zulip
 RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
 RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bash_profile
 WORKDIR /srv/zulip
--- a/42
+++ b/42
@@ -1,42 +0,0 @@
 # To build run `docker build -f Dockerfile-postgresql .` from the root of the
 # zulip repo.
 # Install build tools and build tsearch_extras for the current postgres
 # version. Currently the postgres images do not support automatic upgrading of
 # the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
 FROM postgres:10
 RUN apt-get update \
    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
        postgresql-server-dev-$PG_MAJOR \
        postgresql-server-dev-all \
        git \
        build-essential \
        fakeroot \
        devscripts
 RUN git clone https://github.com/zulip/tsearch_extras.git \
    && cd tsearch_extras \
    && echo $PG_MAJOR > debian/pgversions \
    && pg_buildext updatecontrol \
    && debuild -b -uc -us
 # Install tsearch_extras, hunspell, zulip stop words, and run zulip database
 # init.
 FROM postgres:10
 ENV TSEARCH_EXTRAS_VERSION=0.4
 ENV TSEARCH_EXTRAS_DEB=postgresql-${PG_MAJOR}-tsearch-extras_${TSEARCH_EXTRAS_VERSION}_amd64.deb
 COPY --from=0 /${TSEARCH_EXTRAS_DEB} /tmp
 COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop
 COPY scripts/setup/postgres-create-db /docker-entrypoint-initdb.d/postgres-create-db.sh
 COPY scripts/setup/pgroonga-debian.asc /tmp
 RUN apt-key add /tmp/pgroonga-debian.asc \
    && echo "deb http://packages.groonga.org/debian/ stretch main" > /etc/apt/sources.list.d/zulip.list \
    && apt-get update \
    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
       hunspell-en-us \
       postgresql-${PG_MAJOR}-pgroonga \
    && DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/${TSEARCH_EXTRAS_DEB} \
    && rm /tmp/${TSEARCH_EXTRAS_DEB} \
    && ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \
    && ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \
    && rm -rf /var/lib/apt/lists/*
--- a/204
+++ b/204
@@ -1,203 +1,5 @@
-Copyright 2011-2018 Dropbox, Inc., Kandra Labs, Inc., and contributors
+Copyright <20> 2012-2013 Zulip, Inc.
-                                 Apache License
+This software is licensed under the Zulip Enterprise License Agreement.
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+Zulip can be reached at support@zulip.com.
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright [yyyy] [name of copyright owner]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
--- a/16
+++ b/16
@@ -1,16 +0,0 @@
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this project except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 The software includes some works released by third parties under other
 free and open source licenses. Those works are redistributed under the
 license terms under which the works were received. For more details,
 see the ``docs/THIRDPARTY`` file included with this distribution.
--- a/README.md
+++ b/README.md
@@ -1,84 +0,0 @@
 # Zulip overview
 Zulip is a powerful, open source group chat application that combines the
 immediacy of real-time chat with the productivity benefits of threaded
 conversations. Zulip is used by open source projects, Fortune 500 companies,
 large standards bodies, and others who need a real-time chat system that
 allows users to easily process hundreds or thousands of messages a day. With
 over 300 contributors merging over 500 commits a month, Zulip is also the
 largest and fastest growing open source group chat project.
 [![CircleCI branch](https://img.shields.io/circleci/project/github/zulip/zulip/master.svg)](https://circleci.com/gh/zulip/zulip)
 [![Travis Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip)
 [![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
 [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
 [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
 [![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
 [mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
 ## Getting started
 Click on the appropriate link below. If nothing seems to apply,
 join us on the
 [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
 and tell us what's up!
 You might be interested in:
 * **Contributing code**. Check out our
  [guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
  to get started. Zulip prides itself on maintaining a clean and
  well-tested codebase, and a stock of hundreds of
  [beginner-friendly issues][beginner-friendly].
 * **Contributing non-code**.
  [Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issue),
  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
  into your language,
  [write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
  for the Zulip blog, or
  [give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
  would love to hear from you, even if you're just trying the product out.
 * **Supporting Zulip**. Advocate for your organization to use Zulip, write a
  review in the mobile app stores, or
  [upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
  product comparison sites.
 * **Checking Zulip out**. The best way to see Zulip in action is to drop by
  the
  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
  also recommend reading Zulip for
  [open source](https://zulipchat.com/for/open-source/), Zulip for
  [companies](https://zulipchat.com/for/companies/), or Zulip for
  [working groups and part time communities](https://zulipchat.com/for/working-groups-and-communities/).
 * **Running a Zulip server**. Setting up a server takes just a couple
  of minutes. Zulip runs on Ubuntu 18.04 Bionic, Ubuntu 16.04 Xenial,
  Ubuntu 14.04 Trusty, and Debian 9 Stretch. The installation process is
  [documented here](https://zulip.readthedocs.io/en/stable/production/install.html).
  Commercial support is available; see <https://zulipchat.com/plans>
  for details.
 * **Using Zulip without setting up a server**. <https://zulipchat.com> offers
  free and commercial hosting.
 * **Applying for a Zulip internship**. Zulip runs internship programs with
  [Outreachy](https://www.outreachy.org/),
  [Google Summer of Code](https://developers.google.com/open-source/gsoc/),
  and the
  [MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram). Zulip
  also participates in
  [Google Code-In](https://developers.google.com/open-source/gci/). More
  information is available
  [here](https://zulip.readthedocs.io/en/latest/overview/contributing.html#internship-programs).
 You may also be interested in reading our [blog](http://blog.zulip.org/) or
 following us on [twitter](https://twitter.com/zulip).
 Zulip is distributed under the
 [Apache 2.0](https://github.com/zulip/zulip/blob/master/LICENSE) license.
 [beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22
--- a/226
+++ b/226
@@ -1,226 +0,0 @@
 # -*- mode: ruby -*-
 VAGRANTFILE_API_VERSION = "2"
 def command?(name)
  `which #{name} > /dev/null 2>&1`
  $?.success?
 end
 if Vagrant::VERSION == "1.8.7" then
    path = `which curl`
    if path.include?('/opt/vagrant/embedded/bin/curl') then
        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 "\
             "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
             "issue before provisioning. See "\
             "https://github.com/mitchellh/vagrant/issues/7997 "\
             "for reference."
        exit
    end
 end
 # Workaround: the lxc-config in vagrant-lxc is incompatible with changes in
 # LXC 2.1.0, found in Ubuntu 17.10 artful.  LXC 2.1.1 (in 18.04 LTS bionic)
 # ignores the old config key, so this will only be needed for artful.
 #
 # vagrant-lxc upstream has an attempted fix:
 #   https://github.com/fgrehm/vagrant-lxc/issues/445
 # but it didn't work in our testing.  This is a temporary issue, so we just
 # hack in a fix: we patch the skeleton `lxc-config` file right in the
 # distribution of the vagrant-lxc "box" we use.  If the user doesn't yet
 # have the box (e.g. on first setup), Vagrant would download it but too
 # late for us to patch it like this; so we prompt them to explicitly add it
 # first and then rerun.
 if ['up', 'provision'].include? ARGV[0]
  if command? "lxc-ls"
    LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
    if LXC_VERSION == "2.1.0"
      lxc_config_file = ENV['HOME'] + "/.vagrant.d/boxes/fgrehm-VAGRANTSLASH-trusty64-lxc/1.2.0/lxc/lxc-config"
      if File.file?(lxc_config_file)
        lines = File.readlines(lxc_config_file)
        deprecated_line = "lxc.pivotdir = lxc_putold\n"
        if lines[1] == deprecated_line
          lines[1] = "# #{deprecated_line}"
          File.open(lxc_config_file, 'w') do |f|
            f.puts(lines)
          end
        end
      else
        puts 'You are running LXC 2.1.0, and fgrehm/trusty64-lxc box is incompatible '\
            "with it by default. First add the box by doing:\n"\
            "  vagrant box add  https://vagrantcloud.com/fgrehm/trusty64-lxc\n"\
            'Once this command succeeds, do "vagrant up" again.'
        exit
      end
    end
  end
 end
 # Workaround: Vagrant removed the atlas.hashicorp.com to
 # vagrantcloud.com redirect in February 2018. The value of
 # DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
 # atlas.hashicorp.com, which means that removal broke the fetching and
 # updating of boxes (since the old URL doesn't work).  See
 # https://github.com/hashicorp/vagrant/issues/9442
 if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
  Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
 end
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  # For LXC. VirtualBox hosts use a different box, described below.
  config.vm.box = "fgrehm/trusty64-lxc"
  # The Zulip development environment runs on 9991 on the guest.
  host_port = 9991
  http_proxy = https_proxy = no_proxy = nil
  host_ip_addr = "127.0.0.1"
  config.vm.synced_folder ".", "/vagrant", disabled: true
  if (/darwin/ =~ RUBY_PLATFORM) != nil
    config.vm.synced_folder ".", "/srv/zulip", type: "nfs",
        linux__nfs_options: ['rw']
    config.vm.network "private_network", type: "dhcp"
  else
    config.vm.synced_folder ".", "/srv/zulip"
  end
  vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
  if File.file?(vagrant_config_file)
    IO.foreach(vagrant_config_file) do |line|
      line.chomp!
      key, value = line.split(nil, 2)
      case key
      when /^([#;]|$)/; # ignore comments
      when "HTTP_PROXY"; http_proxy = value
      when "HTTPS_PROXY"; https_proxy = value
      when "NO_PROXY"; no_proxy = value
      when "HOST_PORT"; host_port = value.to_i
      when "HOST_IP_ADDR"; host_ip_addr = value
      end
    end
  end
  if Vagrant.has_plugin?("vagrant-proxyconf")
    if !http_proxy.nil?
      config.proxy.http = http_proxy
    end
    if !https_proxy.nil?
      config.proxy.https = https_proxy
    end
    if !no_proxy.nil?
      config.proxy.no_proxy = no_proxy
    end
  elsif !http_proxy.nil? or !https_proxy.nil?
    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
    # We haven't figured out a workaround.
    puts 'You have specified value for proxy in ~/.zulip-vagrant-config file but did not ' \
         'install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install ' \
         'vagrant-proxyconf` in a terminal.  This error will appear twice.'
    exit
  end
  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
  # Specify LXC provider before VirtualBox provider so it's preferred.
  config.vm.provider "lxc" do |lxc|
    if command? "lxc-ls"
      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
      if LXC_VERSION >= "1.1.0" and LXC_VERSION < "3.0.0"
        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
        # see https://github.com/fgrehm/vagrant-lxc/issues/333
        lxc.customize 'aa_allow_incomplete', 1
      end
      if LXC_VERSION >= "3.0.0"
        lxc.customize 'apparmor.allow_incomplete', 1
      end
      if LXC_VERSION >= "2.0.0"
        lxc.backingstore = 'dir'
      end
    end
  end
  config.vm.provider "virtualbox" do |vb, override|
    override.vm.box = "ubuntu/trusty64"
    # It's possible we can get away with just 1.5GB; more testing needed
    vb.memory = 2048
    vb.cpus = 2
  end
  config.vm.provider "vmware_fusion" do |vb, override|
    override.vm.box = "puphpet/ubuntu1404-x64"
    vb.vmx["memsize"] = "2048"
    vb.vmx["numvcpus"] = "2"
  end
 $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
 # Code should go here, rather than tools/provision, only if it is
 # something that we don't want to happen when running provision in a
 # development environment not using Vagrant.
 # Set the MOTD on the system to have Zulip instructions
 sudo rm -f /etc/update-motd.d/*
 sudo bash -c 'cat << EndOfMessage > /etc/motd
 Welcome to the Zulip development environment!  Popular commands:
 * tools/provision - Update the development environment
 * tools/run-dev.py - Run the development server
 * tools/lint - Run the linter (quick and catches many problems)
 * tools/test-* - Run tests (use --help to learn about options)
 Read https://zulip.readthedocs.io/en/latest/testing/testing.html to learn
 how to run individual test suites so that you can get a fast debug cycle.
 EndOfMessage'
 # If the host is running SELinux remount the /sys/fs/selinux directory as read only,
 # needed for apt-get to work.
 if [ -d "/sys/fs/selinux" ]; then
    sudo mount -o remount,ro /sys/fs/selinux
 fi
 # Set default locale, this prevents errors if the user has another locale set.
 if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
    echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
 fi
 # Set an environment variable, so that we won't print the virtualenv
 # shell warning (it'll be wrong, since the shell is dying anyway)
 export SKIP_VENV_SHELL_WARNING=1
 # End `set -x`, so that the end of provision doesn't look like an error
 # message after a successful run.
 set +x
 # Check if the zulip directory is writable
 if [ ! -w /srv/zulip ]; then
    echo "The vagrant user is unable to write to the zulip directory."
    echo "To fix this, run the following commands on the host machine:"
    # sudo is required since our uid is not 1000
    echo '    vagrant halt -f'
    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
    echo '    sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
    echo "You can resume setting up your vagrant environment by running:"
    echo "    vagrant up"
    exit 1
 fi
 # Provision the development environment
 ln -nsf /srv/zulip ~/zulip
 /srv/zulip/tools/provision
 # Run any custom provision hooks the user has configured
 if [ -f /srv/zulip/tools/custom_provision ]; then
    chmod +x /srv/zulip/tools/custom_provision
    /srv/zulip/tools/custom_provision
 fi
 SCRIPT
  config.vm.provision "shell",
    # We want provision to be run with the permissions of the vagrant user.
    privileged: false,
    inline: $provision_script
 end
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -1,538 +0,0 @@
 import time
 from collections import OrderedDict, defaultdict
 from datetime import datetime, timedelta
 import logging
 from typing import Callable, Dict, List, \
    Optional, Tuple, Type, Union
 from django.conf import settings
 from django.db import connection
 from django.db.models import F
 from analytics.models import BaseCount, \
    FillState, InstallationCount, RealmCount, StreamCount, \
    UserCount, installation_epoch, last_successful_fill
 from zerver.lib.logging_util import log_to_file
 from zerver.lib.timestamp import ceiling_to_day, \
    ceiling_to_hour, floor_to_hour, verify_UTC
 from zerver.models import Message, Realm, \
    Stream, UserActivityInterval, UserProfile, models
 ## Logging setup ##
 logger = logging.getLogger('zulip.management')
 log_to_file(logger, settings.ANALYTICS_LOG_PATH)
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365*1000)
 ## Class definitions ##
 class CountStat:
    HOUR = 'hour'
    DAY = 'day'
    FREQUENCIES = frozenset([HOUR, DAY])
    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
                 interval: Optional[timedelta]=None) -> None:
        self.property = property
        self.data_collector = data_collector
        # might have to do something different for bitfields
        if frequency not in self.FREQUENCIES:
            raise AssertionError("Unknown frequency: %s" % (frequency,))
        self.frequency = frequency
        if interval is not None:
            self.interval = interval
        elif frequency == CountStat.HOUR:
            self.interval = timedelta(hours=1)
        else:  # frequency == CountStat.DAY
            self.interval = timedelta(days=1)
    def __str__(self) -> str:
        return "<CountStat: %s>" % (self.property,)
 class LoggingCountStat(CountStat):
    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 class DependentCountStat(CountStat):
    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
                 interval: Optional[timedelta]=None, dependencies: List[str]=[]) -> None:
        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
        self.dependencies = dependencies
 class DataCollector:
    def __init__(self, output_table: Type[BaseCount],
                 pull_function: Optional[Callable[[str, datetime, datetime], int]]) -> None:
        self.output_table = output_table
        self.pull_function = pull_function
 ## CountStat-level operations ##
 def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
    if stat.frequency == CountStat.HOUR:
        time_increment = timedelta(hours=1)
    elif stat.frequency == CountStat.DAY:
        time_increment = timedelta(days=1)
    else:
        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
    verify_UTC(fill_to_time)
    if floor_to_hour(fill_to_time) != fill_to_time:
        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
    fill_state = FillState.objects.filter(property=stat.property).first()
    if fill_state is None:
        currently_filled = installation_epoch()
        fill_state = FillState.objects.create(property=stat.property,
                                              end_time=currently_filled,
                                              state=FillState.DONE)
        logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
    elif fill_state.state == FillState.STARTED:
        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
        do_delete_counts_at_hour(stat, fill_state.end_time)
        currently_filled = fill_state.end_time - time_increment
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        logger.info("UNDO DONE %s" % (stat.property,))
    elif fill_state.state == FillState.DONE:
        currently_filled = fill_state.end_time
    else:
        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
    if isinstance(stat, DependentCountStat):
        for dependency in stat.dependencies:
            dependency_fill_time = last_successful_fill(dependency)
            if dependency_fill_time is None:
                logger.warning("DependentCountStat %s run before dependency %s." %
                               (stat.property, dependency))
                return
            fill_to_time = min(fill_to_time, dependency_fill_time)
    currently_filled = currently_filled + time_increment
    while currently_filled <= fill_to_time:
        logger.info("START %s %s" % (stat.property, currently_filled))
        start = time.time()
        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
        do_fill_count_stat_at_hour(stat, currently_filled)
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        end = time.time()
        currently_filled = currently_filled + time_increment
        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
 def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
    fill_state.end_time = end_time
    fill_state.state = state
    fill_state.save()
 # We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
 # and is timezone aware. It is the caller's responsibility to enforce this!
 def do_fill_count_stat_at_hour(stat: CountStat, end_time: datetime) -> None:
    start_time = end_time - stat.interval
    if not isinstance(stat, LoggingCountStat):
        timer = time.time()
        assert(stat.data_collector.pull_function is not None)
        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
        logger.info("%s run pull_function (%dms/%sr)" %
                    (stat.property, (time.time()-timer)*1000, rows_added))
    do_aggregate_to_summary_table(stat, end_time)
 def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
    if isinstance(stat, LoggingCountStat):
        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
        if stat.data_collector.output_table in [UserCount, StreamCount]:
            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
    else:
        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
 def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
    cursor = connection.cursor()
    # Aggregate into RealmCount
    output_table = stat.data_collector.output_table
    if output_table in (UserCount, StreamCount):
        realmcount_query = """
            INSERT INTO analytics_realmcount
                (realm_id, value, property, subgroup, end_time)
            SELECT
                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
                %(output_table)s.subgroup, %%(end_time)s
            FROM zerver_realm
            JOIN %(output_table)s
            ON
                zerver_realm.id = %(output_table)s.realm_id
            WHERE
                %(output_table)s.property = '%(property)s' AND
                %(output_table)s.end_time = %%(end_time)s
            GROUP BY zerver_realm.id, %(output_table)s.subgroup
        """ % {'output_table': output_table._meta.db_table,
               'property': stat.property}
        start = time.time()
        cursor.execute(realmcount_query, {'end_time': end_time})
        end = time.time()
        logger.info("%s RealmCount aggregation (%dms/%sr)" % (
            stat.property, (end - start) * 1000, cursor.rowcount))
    # Aggregate into InstallationCount
    installationcount_query = """
        INSERT INTO analytics_installationcount
            (value, property, subgroup, end_time)
        SELECT
            sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
        FROM analytics_realmcount
        WHERE
            property = '%(property)s' AND
            end_time = %%(end_time)s
        GROUP BY analytics_realmcount.subgroup
    """ % {'property': stat.property}
    start = time.time()
    cursor.execute(installationcount_query, {'end_time': end_time})
    end = time.time()
    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (
        stat.property, (end - start) * 1000, cursor.rowcount))
    cursor.close()
 ## Utility functions called from outside counts.py ##
 # called from zerver/lib/actions.py; should not throw any errors
 def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream], stat: CountStat,
                              subgroup: Optional[Union[str, int, bool]], event_time: datetime,
                              increment: int=1) -> None:
    table = stat.data_collector.output_table
    if table == RealmCount:
        id_args = {'realm': zerver_object}
    elif table == UserCount:
        id_args = {'realm': zerver_object.realm, 'user': zerver_object}
    else:  # StreamCount
        id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
    if stat.frequency == CountStat.DAY:
        end_time = ceiling_to_day(event_time)
    else:  # CountStat.HOUR:
        end_time = ceiling_to_hour(event_time)
    row, created = table.objects.get_or_create(
        property=stat.property, subgroup=subgroup, end_time=end_time,
        defaults={'value': increment}, **id_args)
    if not created:
        row.value = F('value') + increment
        row.save(update_fields=['value'])
 def do_drop_all_analytics_tables() -> None:
    UserCount.objects.all().delete()
    StreamCount.objects.all().delete()
    RealmCount.objects.all().delete()
    InstallationCount.objects.all().delete()
    FillState.objects.all().delete()
 def do_drop_single_stat(property: str) -> None:
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 ## DataCollector-level operations ##
 def do_pull_by_sql_query(property: str, start_time: datetime, end_time: datetime, query: str,
                         group_by: Optional[Tuple[models.Model, str]]) -> int:
    if group_by is None:
        subgroup = 'NULL'
        group_by_clause  = ''
    else:
        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
        group_by_clause = ', ' + subgroup
    # We do string replacement here because cursor.execute will reject a
    # group_by_clause given as a param.
    # We pass in the datetimes as params to cursor.execute so that we don't have to
    # think about how to convert python datetimes to SQL datetimes.
    query_ = query % {'property': property, 'subgroup': subgroup,
                      'group_by_clause': group_by_clause}
    cursor = connection.cursor()
    cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
    rowcount = cursor.rowcount
    cursor.close()
    return rowcount
 def sql_data_collector(output_table: Type[BaseCount], query: str,
                       group_by: Optional[Tuple[models.Model, str]]) -> DataCollector:
    def pull_function(property: str, start_time: datetime, end_time: datetime) -> int:
        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
    return DataCollector(output_table, pull_function)
 def do_pull_minutes_active(property: str, start_time: datetime, end_time: datetime) -> int:
    user_activity_intervals = UserActivityInterval.objects.filter(
        end__gt=start_time, start__lt=end_time
    ).select_related(
        'user_profile'
    ).values_list(
        'user_profile_id', 'user_profile__realm_id', 'start', 'end')
    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
        start = max(start_time, interval_start)
        end = min(end_time, interval_end)
        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
    rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
                      end_time=end_time, value=int(seconds // 60))
            for ids, seconds in seconds_active.items() if seconds >= 60]
    UserCount.objects.bulk_create(rows)
    return len(rows)
 count_message_by_user_query = """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
        '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_userprofile
    JOIN zerver_message
    ON
        zerver_userprofile.id = zerver_message.sender_id
    WHERE
        zerver_userprofile.date_joined < %%(time_end)s AND
        zerver_message.pub_date >= %%(time_start)s AND
        zerver_message.pub_date < %%(time_end)s
    GROUP BY zerver_userprofile.id %(group_by_clause)s
 """
 # Note: ignores the group_by / group_by_clause.
 count_message_type_by_user_query = """
    INSERT INTO analytics_usercount
            (realm_id, user_id, value, property, subgroup, end_time)
    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
    FROM
    (
        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
        CASE WHEN
                  zerver_recipient.type = 1 THEN 'private_message'
             WHEN
                  zerver_recipient.type = 3 THEN 'huddle_message'
             WHEN
                  zerver_stream.invite_only = TRUE THEN 'private_stream'
             ELSE 'public_stream'
        END
        message_type
        FROM zerver_userprofile
        JOIN zerver_message
        ON
            zerver_userprofile.id = zerver_message.sender_id AND
            zerver_message.pub_date >= %%(time_start)s AND
            zerver_message.pub_date < %%(time_end)s
        JOIN zerver_recipient
        ON
            zerver_message.recipient_id = zerver_recipient.id
        LEFT JOIN zerver_stream
        ON
            zerver_recipient.type_id = zerver_stream.id
        GROUP BY
            zerver_userprofile.realm_id, zerver_userprofile.id,
            zerver_recipient.type, zerver_stream.invite_only
    ) AS subquery
    GROUP BY realm_id, id, message_type
 """
 # This query joins to the UserProfile table since all current queries that
 # use this also subgroup on UserProfile.is_bot. If in the future there is a
 # stat that counts messages by stream and doesn't need the UserProfile
 # table, consider writing a new query for efficiency.
 count_message_by_stream_query = """
    INSERT INTO analytics_streamcount
        (stream_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_stream
    JOIN zerver_recipient
    ON
        zerver_stream.id = zerver_recipient.type_id
    JOIN zerver_message
    ON
        zerver_recipient.id = zerver_message.recipient_id
    JOIN zerver_userprofile
    ON
        zerver_message.sender_id = zerver_userprofile.id
    WHERE
        zerver_stream.date_created < %%(time_end)s AND
        zerver_recipient.type = 2 AND
        zerver_message.pub_date >= %%(time_start)s AND
        zerver_message.pub_date < %%(time_end)s
    GROUP BY zerver_stream.id %(group_by_clause)s
 """
 # Hardcodes the query needed by active_users:is_bot:day, since that is
 # currently the only stat that uses this.
 count_user_by_realm_query = """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_realm
    JOIN zerver_userprofile
    ON
        zerver_realm.id = zerver_userprofile.realm_id
    WHERE
        zerver_realm.date_created < %%(time_end)s AND
        zerver_userprofile.date_joined >= %%(time_start)s AND
        zerver_userprofile.date_joined < %%(time_end)s AND
        zerver_userprofile.is_active = TRUE
    GROUP BY zerver_realm.id %(group_by_clause)s
 """
 # Currently hardcodes the query needed for active_users_audit:is_bot:day.
 # Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
 # event_type in ['user_created', 'user_deactivated', etc].
 # In particular, it's important to ensure that migrations don't cause that to happen.
 check_realmauditlog_by_user_query = """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_realmauditlog ral1
    JOIN (
        SELECT modified_user_id, max(event_time) AS max_event_time
        FROM zerver_realmauditlog
        WHERE
            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
            event_time < %%(time_end)s
        GROUP BY modified_user_id
    ) ral2
    ON
        ral1.event_time = max_event_time AND
        ral1.modified_user_id = ral2.modified_user_id
    JOIN zerver_userprofile
    ON
        ral1.modified_user_id = zerver_userprofile.id
    WHERE
        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
 """
 check_useractivityinterval_by_user_query = """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_userprofile
    JOIN zerver_useractivityinterval
    ON
        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
    WHERE
        zerver_useractivityinterval.end >= %%(time_start)s AND
        zerver_useractivityinterval.start < %%(time_end)s
    GROUP BY zerver_userprofile.id %(group_by_clause)s
 """
 count_realm_active_humans_query = """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
    FROM (
        SELECT realm_id, user_id
        FROM analytics_usercount
        WHERE
            property = 'active_users_audit:is_bot:day' AND
            subgroup = 'false' AND
            end_time = %%(time_end)s
    ) usercount1
    JOIN (
        SELECT realm_id, user_id
        FROM analytics_usercount
        WHERE
            property = '15day_actives::day' AND
            end_time = %%(time_end)s
    ) usercount2
    ON
        usercount1.user_id = usercount2.user_id
    GROUP BY usercount1.realm_id
 """
 # Currently unused and untested
 count_stream_by_realm_query = """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_realm.id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
    FROM zerver_realm
    JOIN zerver_stream
    ON
        zerver_realm.id = zerver_stream.realm_id AND
    WHERE
        zerver_realm.date_created < %%(time_end)s AND
        zerver_stream.date_created >= %%(time_start)s AND
        zerver_stream.date_created < %%(time_end)s
    GROUP BY zerver_realm.id %(group_by_clause)s
 """
 ## CountStat declarations ##
 count_stats_ = [
    # Messages Sent stats
    # Stats that count the number of messages sent in various ways.
    # These are also the set of stats that read from the Message table.
    CountStat('messages_sent:is_bot:hour',
              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
              CountStat.HOUR),
    CountStat('messages_sent:message_type:day',
              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
    CountStat('messages_sent:client:day',
              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
              CountStat.DAY),
    CountStat('messages_in_stream:is_bot:day',
              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
              CountStat.DAY),
    # Number of Users stats
    # Stats that count the number of active users in the UserProfile.is_active sense.
    # 'active_users_audit:is_bot:day' is the canonical record of which users were
    # active on which days (in the UserProfile.is_active sense).
    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
    CountStat('active_users_audit:is_bot:day',
              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
              CountStat.DAY),
    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
    # sum sequence of 'active_users_log:is_bot:day', for any realm that
    # started after the latter stat was introduced.
    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
    # approximation, e.g. if a user is deactivated between the end of the
    # day and when this stat is run, they won't be counted. However, is the
    # simplest of the three to inspect by hand.
    CountStat('active_users:is_bot:day',
              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
              CountStat.DAY, interval=TIMEDELTA_MAX),
    # User Activity stats
    # Stats that measure user activity in the UserActivityInterval sense.
    CountStat('1day_actives::day',
              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
              CountStat.DAY, interval=timedelta(days=1)-UserActivityInterval.MIN_INTERVAL_LENGTH),
    CountStat('15day_actives::day',
              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
    # Rate limiting stats
    # Used to limit the number of invitation emails sent by a realm
    LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
    # Dependent stats
    # Must come after their dependencies.
    # Canonical account of the number of active humans in a realm on each day.
    DependentCountStat('realm_active_humans::day',
                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
                       CountStat.DAY,
                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
 ]
 COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
--- a/analytics/lib/fixtures.py
+++ b/analytics/lib/fixtures.py
@@ -1,63 +0,0 @@
 from math import sqrt
 from random import gauss, random, seed
 from typing import List
 from analytics.lib.counts import CountStat
 def generate_time_series_data(days: int=100, business_hours_base: float=10,
                              non_business_hours_base: float=10, growth: float=1,
                              autocorrelation: float=0, spikiness: float=1,
                              holiday_rate: float=0, frequency: str=CountStat.DAY,
                              partial_sum: bool=False, random_seed: int=26) -> List[int]:
    """
    Generate semi-realistic looking time series data for testing analytics graphs.
    days -- Number of days of data. Is the number of data points generated if
        frequency is CountStat.DAY.
    business_hours_base -- Average value during a business hour (or day) at beginning of
        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
    non_business_hours_base -- The above, for non-business hours/days.
    growth -- Ratio between average values at end of time series and beginning of time series.
    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
        point is unaffected by the previous point, and at 1 each point is a deterministic
        function of the previous point.
    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
        the variance.
    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
    frequency -- Should be CountStat.HOUR or CountStat.DAY.
    partial_sum -- If True, return partial sum of the series.
    random_seed -- Seed for random number generator.
    """
    if frequency == CountStat.HOUR:
        length = days*24
        seasonality = [non_business_hours_base] * 24 * 7
        for day in range(5):
            for hour in range(8):
                seasonality[24*day + hour] = business_hours_base
        holidays  = []
        for i in range(days):
            holidays.extend([random() < holiday_rate] * 24)
    elif frequency == CountStat.DAY:
        length = days
        seasonality = [8*business_hours_base + 16*non_business_hours_base] * 5 + \
                      [24*non_business_hours_base] * 2
        holidays = [random() < holiday_rate for i in range(days)]
    else:
        raise AssertionError("Unknown frequency: %s" % (frequency,))
    if length < 2:
        raise AssertionError("Must be generating at least 2 data points. "
                             "Currently generating %s" % (length,))
    growth_base = growth ** (1. / (length-1))
    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
    seed(random_seed)
    noise_scalars = [gauss(0, 1)]
    for i in range(1, length):
        noise_scalars.append(noise_scalars[-1]*autocorrelation + gauss(0, 1)*(1-autocorrelation))
    values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
              for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
    if partial_sum:
        for i in range(1, length):
            values[i] = values[i-1] + values[i]
    return [max(v, 0) for v in values]
--- a/analytics/lib/time_utils.py
+++ b/analytics/lib/time_utils.py
@@ -1,31 +0,0 @@
 from datetime import datetime, timedelta
 from typing import List, Optional
 from analytics.lib.counts import CountStat
 from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
 def time_range(start: datetime, end: datetime, frequency: str,
               min_length: Optional[int]) -> List[datetime]:
    verify_UTC(start)
    verify_UTC(end)
    if frequency == CountStat.HOUR:
        end = floor_to_hour(end)
        step = timedelta(hours=1)
    elif frequency == CountStat.DAY:
        end = floor_to_day(end)
        step = timedelta(days=1)
    else:
        raise AssertionError("Unknown frequency: %s" % (frequency,))
    times = []
    if min_length is not None:
        start = min(start, end - (min_length-1)*step)
    current = end
    while current >= start:
        times.append(current)
        current -= step
    return list(reversed(times))
--- a/analytics/management/commands/active_user_stats.py
+++ b/analytics/management/commands/active_user_stats.py
@@ -0,0 +1,57 @@
 from __future__ import absolute_import
 from django.core.management.base import BaseCommand
 from zerver.models import UserPresence, UserActivity
 from zerver.lib.utils import statsd, statsd_key
 from datetime import datetime, timedelta
 from collections import defaultdict
 class Command(BaseCommand):
    help = """Sends active user statistics to statsd.
    Run as a cron job that runs every 10 minutes."""
    def handle(self, *args, **options):
        # Get list of all active users in the last 1 week
        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
        hour_buckets = [0.16, 2, 12, 24, 48, 168]
        user_info = defaultdict(dict)
        for last_presence in users:
            if last_presence.status == UserPresence.IDLE:
                known_active = last_presence.timestamp - timedelta(minutes=30)
            else:
                known_active = last_presence.timestamp
            for bucket in hour_buckets:
                if not bucket in user_info[last_presence.user_profile.realm.domain]:
                    user_info[last_presence.user_profile.realm.domain][bucket] = []
                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
                    user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
        for realm, buckets in user_info.items():
            print("Realm %s" % realm)
            for hr, users in sorted(buckets.items()):
                print("\tUsers for %s: %s" % (hr, len(users)))
                statsd.gauge("users.active.%s.%shr" %  (statsd_key(realm, True), statsd_key(hr, True)), len(users))
        # Also do stats for how many users have been reading the app.
        users_reading = UserActivity.objects.select_related().filter(query="/json/update_message_flags")
        user_info = defaultdict(dict)
        for activity in users_reading:
            for bucket in hour_buckets:
                if not bucket in user_info[activity.user_profile.realm.domain]:
                    user_info[activity.user_profile.realm.domain][bucket] = []
                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
                    user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
        for realm, buckets in user_info.items():
            print("Realm %s" % realm)
            for hr, users in sorted(buckets.items()):
                print("\tUsers reading for %s: %s" % (hr, len(users)))
                statsd.gauge("users.reading.%s.%shr" %  (statsd_key(realm, True), statsd_key(hr, True)), len(users))
--- a/analytics/management/commands/active_user_stats_by_day.py
+++ b/analytics/management/commands/active_user_stats_by_day.py
@@ -0,0 +1,24 @@
 from __future__ import absolute_import
 import datetime
 import pytz
 from optparse import make_option
 from django.core.management.base import BaseCommand
 from zerver.lib.statistics import activity_averages_during_day
 class Command(BaseCommand):
    help = "Generate statistics on user activity for a given day."
    option_list = BaseCommand.option_list + \
        (make_option('--date', default=None, action='store',
                     help="Day to query in format 2013-12-05.  Default is yesterday"),)
    def handle(self, *args, **options):
        if options["date"] is None:
            date = datetime.datetime.now() - datetime.timedelta(days=1)
        else:
            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
        print "Activity data for", date
        print activity_averages_during_day(date)
        print "Please note that the total registered user count is a total for today"
--- a/analytics/management/commands/analyze_mit.py
+++ b/analytics/management/commands/analyze_mit.py
@@ -1,19 +1,19 @@
-import datetime
+from __future__ import absolute_import
 import logging
 import time
 from typing import Any, Dict
 from django.core.management.base import BaseCommand, CommandParser
 from optparse import make_option
 from django.core.management.base import BaseCommand
 from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Message, Recipient
+import datetime
 import time
 import logging
-def compute_stats(log_level: int) -> None:
+def compute_stats(log_level):
    logger = logging.getLogger()
    logger.setLevel(log_level)
    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
-    mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
+    mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
                                       recipient__type=Recipient.STREAM,
                                       pub_date__gt=one_week_ago)
    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
@@ -26,15 +26,15 @@ def compute_stats(log_level: int) -> None:
                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                           "root@mit.edu", "nagios@mit.edu",
                           "www-data|local-realm@mit.edu"])
-    user_counts = {}  # type: Dict[str, Dict[str, int]]
+    user_counts = {}
    for m in mit_query.select_related("sending_client", "sender"):
        email = m.sender.email
        user_counts.setdefault(email, {})
        user_counts[email].setdefault(m.sending_client.name, 0)
        user_counts[email][m.sending_client.name] += 1
-    total_counts = {}  # type: Dict[str, int]
+    total_counts = {}
-    total_user_counts = {}  # type: Dict[str, int]
+    total_user_counts = {}
    for email, counts in user_counts.items():
        total_user_counts.setdefault(email, 0)
        for client_name, count in counts.items():
@@ -43,9 +43,9 @@ def compute_stats(log_level: int) -> None:
            total_user_counts[email] += count
    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
-    top_percents = {}  # type: Dict[int, float]
+    top_percents = {}
    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
-        top_percents[size] = 0.0
+        top_percents[size] = 0
    for i, email in enumerate(sorted(total_user_counts.keys(),
                                     key=lambda x: -total_user_counts[x])):
        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
@@ -63,18 +63,18 @@ def compute_stats(log_level: int) -> None:
        logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
    grand_total = sum(total_counts.values())
-    print(grand_total)
+    print grand_total
    logging.info("%15s | %s" % ("Client", "Percentage"))
    for client in total_counts.keys():
        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
 class Command(BaseCommand):
    option_list = BaseCommand.option_list + \
        (make_option('--verbose', default=False, action='store_true'),)
    help = "Compute statistics on MIT Zephyr usage."
-    def add_arguments(self, parser: CommandParser) -> None:
+    def handle(self, *args, **options):
        parser.add_argument('--verbose', default=False, action='store_true')
    def handle(self, *args: Any, **options: Any) -> None:
        level = logging.INFO
        if options["verbose"]:
            level = logging.DEBUG
--- a/analytics/management/commands/analyze_user_activity.py
+++ b/analytics/management/commands/analyze_user_activity.py
@@ -1,21 +1,22 @@
-import datetime
+from __future__ import absolute_import
 from typing import Any, Dict
 from django.core.management.base import BaseCommand, CommandParser
 from django.utils.timezone import utc
 from zerver.lib.statistics import seconds_usage_between
 from zerver.models import UserProfile
-def analyze_activity(options: Dict[str, Any]) -> None:
+from optparse import make_option
 from django.core.management.base import BaseCommand
 from zerver.models import UserProfile
 import datetime
 from django.utils.timezone import utc
 def analyze_activity(options):
    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
    day_end = day_start + datetime.timedelta(days=options["duration"])
    user_profile_query = UserProfile.objects.all()
    if options["realm"]:
-        user_profile_query = user_profile_query.filter(realm__string_id=options["realm"])
+        user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
-    print("Per-user online duration:\n")
+    print "Per-user online duration:\n"
    total_duration = datetime.timedelta(0)
    for user_profile in user_profile_query:
        duration = seconds_usage_between(user_profile, day_start, day_end)
@@ -24,11 +25,11 @@ def analyze_activity(options: Dict[str, Any]) -> None:
            continue
        total_duration += duration
-        print("%-*s%s" % (37, user_profile.email, duration,))
+        print "%-*s%s" % (37, user_profile.email, duration, )
-    print("\nTotal Duration:                      %s" % (total_duration,))
+    print "\nTotal Duration:                      %s" % (total_duration,)
-    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
+    print "\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,)
-    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
+    print "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
 class Command(BaseCommand):
    help = """Report analytics of user activity on a per-user and realm basis.
@@ -41,16 +42,16 @@ It will correctly not count server-initiated reloads in the activity statistics.
 The duration flag can be used to control how many days to show usage duration for
-Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--duration=1]
+Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""
-    def add_arguments(self, parser: CommandParser) -> None:
+    option_list = BaseCommand.option_list + (
-        parser.add_argument('--realm', action='store')
+        make_option('--realm', action='store'),
-        parser.add_argument('--date', action='store', default="2013-09-06")
+        make_option('--date', action='store', default="2013-09-06"),
-        parser.add_argument('--duration', action='store', default=1, type=int,
+        make_option('--duration', action='store', default=1, type=int, help="How many days to show usage information for"),
-                            help="How many days to show usage information for")
+        )
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
        analyze_activity(options)
--- a/analytics/management/commands/check_analytics_state.py
+++ b/analytics/management/commands/check_analytics_state.py
@@ -1,82 +0,0 @@
 from datetime import timedelta
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 from analytics.models import installation_epoch, \
    last_successful_fill
 from analytics.lib.counts import COUNT_STATS, CountStat
 from zerver.lib.timestamp import floor_to_hour, floor_to_day, verify_UTC, \
    TimezoneNotUTCException
 from zerver.models import Realm
 import os
 import time
 from typing import Any, Dict
 states = {
    0: "OK",
    1: "WARNING",
    2: "CRITICAL",
    3: "UNKNOWN"
 }
 class Command(BaseCommand):
    help = """Checks FillState table.
    Run as a cron job that runs every hour."""
    def handle(self, *args: Any, **options: Any) -> None:
        fill_state = self.get_fill_state()
        status = fill_state['status']
        message = fill_state['message']
        state_file_path = "/var/lib/nagios_state/check-analytics-state"
        state_file_tmp = state_file_path + "-tmp"
        with open(state_file_tmp, "w") as f:
            f.write("%s|%s|%s|%s\n" % (
                int(time.time()), status, states[status], message))
        os.rename(state_file_tmp, state_file_path)
    def get_fill_state(self) -> Dict[str, Any]:
        if not Realm.objects.exists():
            return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
        warning_unfilled_properties = []
        critical_unfilled_properties = []
        for property, stat in COUNT_STATS.items():
            last_fill = last_successful_fill(property)
            if last_fill is None:
                last_fill = installation_epoch()
            try:
                verify_UTC(last_fill)
            except TimezoneNotUTCException:
                return {'status': 2, 'message': 'FillState not in UTC for %s' % (property,)}
            if stat.frequency == CountStat.DAY:
                floor_function = floor_to_day
                warning_threshold = timedelta(hours=26)
                critical_threshold = timedelta(hours=50)
            else:  # CountStat.HOUR
                floor_function = floor_to_hour
                warning_threshold = timedelta(minutes=90)
                critical_threshold = timedelta(minutes=150)
            if floor_function(last_fill) != last_fill:
                return {'status': 2, 'message': 'FillState not on %s boundary for %s' %
                        (stat.frequency, property)}
            time_to_last_fill = timezone_now() - last_fill
            if time_to_last_fill > critical_threshold:
                critical_unfilled_properties.append(property)
            elif time_to_last_fill > warning_threshold:
                warning_unfilled_properties.append(property)
        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
            return {'status': 0, 'message': 'FillState looks fine.'}
        if len(critical_unfilled_properties) == 0:
            return {'status': 1, 'message': 'Missed filling %s once.' %
                    (', '.join(warning_unfilled_properties),)}
        return {'status': 2, 'message': 'Missed filling %s once. Missed filling %s at least twice.' %
                (', '.join(warning_unfilled_properties), ', '.join(critical_unfilled_properties))}
--- a/analytics/management/commands/clear_analytics_tables.py
+++ b/analytics/management/commands/clear_analytics_tables.py
@@ -1,22 +0,0 @@
 import sys
 from argparse import ArgumentParser
 from typing import Any
 from django.core.management.base import BaseCommand
 from analytics.lib.counts import do_drop_all_analytics_tables
 class Command(BaseCommand):
    help = """Clear analytics tables."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument('--force',
                            action='store_true',
                            help="Clear analytics tables.")
    def handle(self, *args: Any, **options: Any) -> None:
        if options['force']:
            do_drop_all_analytics_tables()
        else:
            print("Would delete all data from analytics tables (!); use --force to do so.")
            sys.exit(1)
--- a/analytics/management/commands/clear_single_stat.py
+++ b/analytics/management/commands/clear_single_stat.py
@@ -1,29 +0,0 @@
 import sys
 from argparse import ArgumentParser
 from typing import Any
 from django.core.management.base import BaseCommand
 from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 class Command(BaseCommand):
    help = """Clear analytics tables."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument('--force',
                            action='store_true',
                            help="Actually do it.")
        parser.add_argument('--property',
                            type=str,
                            help="The property of the stat to be cleared.")
    def handle(self, *args: Any, **options: Any) -> None:
        property = options['property']
        if property not in COUNT_STATS:
            print("Invalid property: %s" % (property,))
            sys.exit(1)
        if not options['force']:
            print("No action taken. Use --force.")
            sys.exit(1)
        do_drop_single_stat(property)
--- a/analytics/management/commands/client_activity.py
+++ b/analytics/management/commands/client_activity.py
@@ -1,32 +1,23 @@
 from __future__ import absolute_import
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from zerver.models import UserActivity, UserProfile, Realm, \
    get_realm, get_user_profile_by_email
 import datetime
 from argparse import ArgumentParser
 from typing import Any
-from django.db.models import Count, QuerySet
+class Command(BaseCommand):
 from django.utils.timezone import now as timezone_now
 from zerver.lib.management import ZulipBaseCommand
 from zerver.models import UserActivity
 class Command(ZulipBaseCommand):
    help = """Report rough client activity globally, for a realm, or for a user
 Usage examples:
-./manage.py client_activity --target server
+python manage.py client_activity
-./manage.py client_activity --target realm --realm zulip
+python manage.py client_activity zulip.com
-./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
+python manage.py client_activity jesstess@zulip.com"""
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def compute_activity(self, user_activity_objects):
        parser.add_argument('--target', dest='target', required=True, type=str,
                            help="'server' will calculate client activity of the entire server. "
                                 "'realm' will calculate client activity of realm. "
                                 "'user' will calculate client activity of the user.")
        parser.add_argument('--user', dest='user', type=str,
                            help="The email address of the user you want to calculate activity.")
        self.add_realm_args(parser)
    def compute_activity(self, user_activity_objects: QuerySet) -> None:
        # Report data from the past week.
        #
        # This is a rough report of client activity because we inconsistently
@@ -37,7 +28,7 @@ Usage examples:
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
-        threshold = timezone_now() - datetime.timedelta(days=7)
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))
@@ -53,21 +44,26 @@ Usage examples:
        counts.sort()
        for count in counts:
-            print("%25s %15d" % (count[1], count[0]))
+            print "%25s %15d" % (count[1], count[0])
-        print("Total:", total)
+        print "Total:", total
-    def handle(self, *args: Any, **options: str) -> None:
+
-        realm = self.get_realm(options)
+    def handle(self, *args, **options):
-        if options["user"] is None:
+        if len(args) == 0:
            if options["target"] == "server" and realm is None:
            # Report global activity.
            self.compute_activity(UserActivity.objects.all())
-            elif options["target"] == "realm" and realm is not None:
+        elif len(args) == 1:
-                self.compute_activity(UserActivity.objects.filter(user_profile__realm=realm))
+            try:
-            else:
+                # Report activity for a user.
-                self.print_help("./manage.py", "client_activity")
+                user_profile = get_user_profile_by_email(args[0])
-        elif options["target"] == "user":
+                self.compute_activity(UserActivity.objects.filter(
-            user_profile = self.get_user(options["user"], realm)
+                        user_profile=user_profile))
-            self.compute_activity(UserActivity.objects.filter(user_profile=user_profile))
+            except UserProfile.DoesNotExist:
-        else:
+                try:
-            self.print_help("./manage.py", "client_activity")
+                    # Report activity for a realm.
                    realm = get_realm(args[0])
                    self.compute_activity(UserActivity.objects.filter(
                            user_profile__realm=realm))
                except Realm.DoesNotExist:
                    print "Unknown user or domain %s" % (args[0],)
                    exit(1)
--- a/analytics/management/commands/populate_analytics_db.py
+++ b/analytics/management/commands/populate_analytics_db.py
@@ -1,224 +0,0 @@
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Mapping, Optional, Type
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, \
    CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
 from analytics.models import BaseCount, FillState, RealmCount, UserCount, \
    StreamCount, InstallationCount
 from zerver.lib.actions import do_change_is_admin, STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, UserProfile, Stream, Client, \
    RealmAuditLog, Recipient, Subscription
 class Command(BaseCommand):
    help = """Populates analytics tables with randomly generated data."""
    DAYS_OF_DATA = 100
    random_seed = 26
    def create_user(self, email: str,
                    full_name: str,
                    is_staff: bool,
                    date_joined: datetime,
                    realm: Realm) -> UserProfile:
        user = UserProfile.objects.create(
            email=email, full_name=full_name, is_staff=is_staff,
            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
            api_key='42', date_joined=date_joined)
        RealmAuditLog.objects.create(
            realm=realm, modified_user=user, event_type=RealmAuditLog.USER_CREATED,
            event_time=user.date_joined)
        return user
    def generate_fixture_data(self, stat: CountStat, business_hours_base: float,
                              non_business_hours_base: float, growth: float,
                              autocorrelation: float, spikiness: float,
                              holiday_rate: float=0, partial_sum: bool=False) -> List[int]:
        self.random_seed += 1
        return generate_time_series_data(
            days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
            non_business_hours_base=non_business_hours_base, growth=growth,
            autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
            frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
    def handle(self, *args: Any, **options: Any) -> None:
        # TODO: This should arguably only delete the objects
        # associated with the "analytics" realm.
        do_drop_all_analytics_tables()
        # This also deletes any objects with this realm as a foreign key
        Realm.objects.filter(string_id='analytics').delete()
        # Because we just deleted a bunch of objects in the database
        # directly (rather than deleting individual objects in Django,
        # in which case our post_save hooks would have flushed the
        # individual objects from memcached for us), we need to flush
        # memcached in order to ensure deleted objects aren't still
        # present in the memcached cache.
        from zerver.apps import flush_cache
        flush_cache(None)
        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
        last_end_time = floor_to_day(timezone_now())
        realm = Realm.objects.create(
            string_id='analytics', name='Analytics', date_created=installation_time)
        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
        do_change_is_admin(shylock, True)
        stream = Stream.objects.create(
            name='all', realm=realm, date_created=installation_time)
        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
        # Subscribe shylock to the stream to avoid invariant failures.
        # TODO: This should use subscribe_users_to_streams from populate_db.
        subs = [
            Subscription(recipient=recipient,
                         user_profile=shylock,
                         color=STREAM_ASSIGNMENT_COLORS[0]),
        ]
        Subscription.objects.bulk_create(subs)
        def insert_fixture_data(stat: CountStat,
                                fixture_data: Mapping[Optional[str], List[int]],
                                table: Type[BaseCount]) -> None:
            end_times = time_range(last_end_time, last_end_time, stat.frequency,
                                   len(list(fixture_data.values())[0]))
            if table == InstallationCount:
                id_args = {}  # type: Dict[str, Any]
            if table == RealmCount:
                id_args = {'realm': realm}
            if table == UserCount:
                id_args = {'realm': realm, 'user': shylock}
            if table == StreamCount:
                id_args = {'stream': stream, 'realm': realm}
            for subgroup, values in fixture_data.items():
                table.objects.bulk_create([
                    table(property=stat.property, subgroup=subgroup, end_time=end_time,
                          value=value, **id_args)
                    for end_time, value in zip(end_times, values) if value != 0])
        stat = COUNT_STATS['1day_actives::day']
        realm_data = {
            None: self.generate_fixture_data(stat, .08, .02, 3, .3, 6, partial_sum=True),
        }  # type: Mapping[Optional[str], List[int]]
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None: self.generate_fixture_data(stat, .8, .2, 4, .3, 6, partial_sum=True),
        }  # type: Mapping[Optional[str], List[int]]
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        stat = COUNT_STATS['realm_active_humans::day']
        realm_data = {
            None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None: self.generate_fixture_data(stat, 1, .3, 4, .5, 3, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        stat = COUNT_STATS['active_users_audit:is_bot:day']
        realm_data = {
            'false': self.generate_fixture_data(stat, .1, .03, 3.5, .8, 2, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            'false': self.generate_fixture_data(stat, 1, .3, 6, .8, 2, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        stat = COUNT_STATS['messages_sent:is_bot:hour']
        user_data = {'false': self.generate_fixture_data(
            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
                      'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {'false': self.generate_fixture_data(stat, 350, 150, 6, .6, 4),
                             'true': self.generate_fixture_data(stat, 150, 150, 3, .4, 2)}
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        stat = COUNT_STATS['messages_sent:message_type:day']
        user_data = {
            'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
            'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
            'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
            'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            'public_stream': self.generate_fixture_data(stat, 300, 80, 5, .6, 4),
            'private_stream': self.generate_fixture_data(stat, 70, 70, 5, .6, 4),
            'private_message': self.generate_fixture_data(stat, 130, 50, 5, .6, 4),
            'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, .6, 4)}
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        website, created = Client.objects.get_or_create(name='website')
        old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
        android, created = Client.objects.get_or_create(name='ZulipAndroid')
        iOS, created = Client.objects.get_or_create(name='ZulipiOS')
        react_native, created = Client.objects.get_or_create(name='ZulipMobile')
        API, created = Client.objects.get_or_create(name='API: Python')
        zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
        unused, created = Client.objects.get_or_create(name='unused')
        long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
        stat = COUNT_STATS['messages_sent:client:day']
        user_data = {
            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
            zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
            android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
            API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            website.id: self.generate_fixture_data(stat, 300, 200, 5, .6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, .6, 3),
            android.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
            API.id: self.generate_fixture_data(stat, 50, 50, 5, .6, 3),
            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, .6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3)}
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
        stat = COUNT_STATS['messages_in_stream:is_bot:day']
        realm_data = {'false': self.generate_fixture_data(stat, 30, 5, 6, .6, 4),
                      'true': self.generate_fixture_data(stat, 20, 2, 3, .2, 3)}
        insert_fixture_data(stat, realm_data, RealmCount)
        stream_data = {'false': self.generate_fixture_data(stat, 10, 7, 5, .6, 4),
                       'true': self.generate_fixture_data(stat, 5, 3, 2, .4, 2)}  # type: Mapping[Optional[str], List[int]]
        insert_fixture_data(stat, stream_data, StreamCount)
        FillState.objects.create(property=stat.property, end_time=last_end_time,
                                 state=FillState.DONE)
--- a/analytics/management/commands/realm_stats.py
+++ b/analytics/management/commands/realm_stats.py
@@ -1,13 +1,12 @@
 from __future__ import absolute_import
 import datetime
-from argparse import ArgumentParser
+import pytz
 from typing import Any, List
 from django.core.management.base import BaseCommand
 from django.db.models import Count
-from django.utils.timezone import now as timezone_now
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
-
+    Subscription, UserMessage
 from zerver.models import Message, Realm, Recipient, Stream, \
    Subscription, UserActivity, UserMessage, UserProfile, get_realm
 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
@@ -17,100 +16,97 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
 class Command(BaseCommand):
    help = "Generate statistics on realm activity."
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def active_users(self, realm):
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")
    def active_users(self, realm: Realm) -> List[UserProfile]:
        # Has been active (on the website, for now) in the last 7 days.
-        activity_cutoff = timezone_now() - datetime.timedelta(days=7)
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
-        return [activity.user_profile for activity in (
+        return [activity.user_profile for activity in \
                    UserActivity.objects.filter(user_profile__realm=realm,
                                                user_profile__is_active=True,
                                                last_visit__gt=activity_cutoff,
-                                        query="/json/users/me/pointer",
+                                                query="/json/update_pointer",
-                                        client__name="website"))]
+                                                client__name="website")]
-    def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
+    def messages_sent_by(self, user, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
-    def total_messages(self, realm: Realm, days_ago: int) -> int:
+    def total_messages(self, realm, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
-    def human_messages(self, realm: Realm, days_ago: int) -> int:
+    def human_messages(self, realm, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
-    def api_messages(self, realm: Realm, days_ago: int) -> int:
+    def api_messages(self, realm, days_ago):
        return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
-    def stream_messages(self, realm: Realm, days_ago: int) -> int:
+    def stream_messages(self, realm, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                     recipient__type=Recipient.STREAM).count()
-    def private_messages(self, realm: Realm, days_ago: int) -> int:
+    def private_messages(self, realm, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
-    def group_private_messages(self, realm: Realm, days_ago: int) -> int:
+    def group_private_messages(self, realm, days_ago):
-        sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
-    def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
+    def report_percentage(self, numerator, denominator, text):
        if not denominator:
            fraction = 0.0
        else:
            fraction = numerator / float(denominator)
-        print("%.2f%% of" % (fraction * 100,), text)
+        print "%.2f%% of" % (fraction * 100,), text
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
-        if options['realms']:
+        if args:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [Realm.objects.get(domain=domain) for domain in args]
-            except Realm.DoesNotExist as e:
+            except Realm.DoesNotExist, e:
-                print(e)
+                print e
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
-            print(realm.string_id)
+            print realm.domain
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            active_users = self.active_users(realm)
            num_active = len(active_users)
-            print("%d active users (%d total)" % (num_active, len(user_profiles)))
+            print "%d active users (%d total)" % (num_active, len(user_profiles))
            streams = Stream.objects.filter(realm=realm).extra(
                tables=['zerver_subscription', 'zerver_recipient'],
                where=['zerver_subscription.recipient_id = zerver_recipient.id',
                       'zerver_recipient.type = 2',
                       'zerver_recipient.type_id = zerver_stream.id',
                       'zerver_subscription.active = true']).annotate(count=Count("name"))
-            print("%d streams" % (streams.count(),))
+            print "%d streams" % (streams.count(),)
            for days_ago in (1, 7, 30):
-                print("In last %d days, users sent:" % (days_ago,))
+                print "In last %d days, users sent:" % (days_ago,)
                sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
                for quantity in sorted(sender_quantities, reverse=True):
-                    print(quantity, end=' ')
+                    print quantity,
-                print("")
+                print ""
-                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
+                print "%d stream messages" % (self.stream_messages(realm, days_ago),)
-                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
+                print "%d one-on-one private messages" % (self.private_messages(realm, days_ago),)
-                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
+                print "%d messages sent via the API" % (self.api_messages(realm, days_ago),)
-                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
+                print "%d group private messages" % (self.group_private_messages(realm, days_ago),)
-            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
+            num_notifications_enabled = len(filter(lambda x: x.enable_desktop_notifications == True,
                                                   active_users))
            self.report_percentage(num_notifications_enabled, num_active,
                                   "active users have desktop notifications enabled")
-            num_enter_sends = len([x for x in active_users if x.enter_sends])
+            num_enter_sends = len(filter(lambda x: x.enter_sends, active_users))
            self.report_percentage(num_enter_sends, num_active,
                                   "active users have enter-sends")
@@ -124,8 +120,8 @@ class Command(BaseCommand):
            starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
                                                  flags=UserMessage.flags.starred).values(
                "user_profile").annotate(count=Count("user_profile"))
-            print("%d users have starred %d messages" % (
+            print "%d users have starred %d messages" % (
-                len(starrers), sum([elt["count"] for elt in starrers])))
+                len(starrers), sum([elt["count"] for elt in starrers]))
            active_user_subs = Subscription.objects.filter(
                user_profile__in=user_profiles, active=True)
@@ -133,20 +129,20 @@ class Command(BaseCommand):
            # Streams not in home view
            non_home_view = active_user_subs.filter(in_home_view=False).values(
                "user_profile").annotate(count=Count("user_profile"))
-            print("%d users have %d streams not in home view" % (
+            print "%d users have %d streams not in home view" % (
-                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
+                len(non_home_view), sum([elt["count"] for elt in non_home_view]))
            # Code block markup
            markup_messages = human_messages.filter(
                sender__realm=realm, content__contains="~~~").values(
                "sender").annotate(count=Count("sender"))
-            print("%d users have used code block markup on %s messages" % (
+            print "%d users have used code block markup on %s messages" % (
-                len(markup_messages), sum([elt["count"] for elt in markup_messages])))
+                len(markup_messages), sum([elt["count"] for elt in markup_messages]))
            # Notifications for stream messages
-            notifications = active_user_subs.filter(desktop_notifications=True).values(
+            notifications = active_user_subs.filter(notifications=True).values(
                "user_profile").annotate(count=Count("user_profile"))
-            print("%d users receive desktop notifications for %d streams" % (
+            print "%d users receive desktop notifications for %d streams" % (
-                len(notifications), sum([elt["count"] for elt in notifications])))
+                len(notifications), sum([elt["count"] for elt in notifications]))
-            print("")
+            print ""
--- a/analytics/management/commands/stream_stats.py
+++ b/analytics/management/commands/stream_stats.py
@@ -1,57 +1,36 @@
-from argparse import ArgumentParser
+from __future__ import absolute_import
 from typing import Any
 from django.core.management.base import BaseCommand
 from django.db.models import Q
-
+from zerver.models import Realm, Stream, Message, Subscription, Recipient
 from zerver.models import Message, Realm, \
    Recipient, Stream, Subscription, get_realm
 class Command(BaseCommand):
    help = "Generate statistics on the streams for a realm."
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def handle(self, *args, **options):
-        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+        if args:
                            help="realm to generate statistics for")
    def handle(self, *args: Any, **options: str) -> None:
        if options['realms']:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [Realm.objects.get(domain=domain) for domain in args]
-            except Realm.DoesNotExist as e:
+            except Realm.DoesNotExist, e:
-                print(e)
+                print e
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
            print realm.domain
            print "------------"
            print "%25s %15s %10s" % ("stream", "subscribers", "messages")
            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
-            # private stream count
+            invite_only_count = 0
            private_count = 0
            # public stream count
            public_count = 0
            for stream in streams:
                if stream.invite_only:
-                    private_count += 1
+                    invite_only_count += 1
-                else:
+                    continue
-                    public_count += 1
+                print "%25s" % (stream.name,),
            print("------------")
            print(realm.string_id, end=' ')
            print("%10s %d public streams and" % ("(", public_count), end=' ')
            print("%d private streams )" % (private_count,))
            print("------------")
            print("%25s %15s %10s %12s" % ("stream", "subscribers", "messages", "type"))
            for stream in streams:
                if stream.invite_only:
                    stream_type = 'private'
                else:
                    stream_type = 'public'
                print("%25s" % (stream.name,), end=' ')
                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
-                print("%10d" % (len(Subscription.objects.filter(recipient=recipient,
+                print "%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),),
                                                                active=True)),), end=' ')
                num_messages = len(Message.objects.filter(recipient=recipient))
-                print("%12d" % (num_messages,), end=' ')
+                print "%12d" % (num_messages,)
-                print("%15s" % (stream_type,))
+            print "%d invite-only streams" % (invite_only_count,)
-            print("")
+            print ""
--- a/analytics/management/commands/update_analytics_counts.py
+++ b/analytics/management/commands/update_analytics_counts.py
@@ -1,90 +0,0 @@
 import os
 import time
 from argparse import ArgumentParser
 from typing import Any, Dict
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
 from django.utils.timezone import utc as timezone_utc
 from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
 from scripts.lib.zulip_tools import ENDC, WARNING
 from zerver.lib.remote_server import send_analytics_to_remote_server
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 class Command(BaseCommand):
    help = """Fills Analytics tables.
    Run as a cron job that runs every hour."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument('--time', '-t',
                            type=str,
                            help='Update stat tables from current state to'
                                 '--time. Defaults to the current time.',
                            default=timezone_now().isoformat())
        parser.add_argument('--utc',
                            action='store_true',
                            help="Interpret --time in UTC.",
                            default=False)
        parser.add_argument('--stat', '-s',
                            type=str,
                            help="CountStat to process. If omitted, all stats are processed.")
        parser.add_argument('--verbose',
                            action='store_true',
                            help="Print timing information to stdout.",
                            default=False)
    def handle(self, *args: Any, **options: Any) -> None:
        try:
            os.mkdir(settings.ANALYTICS_LOCK_DIR)
        except OSError:
            print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
            return
        try:
            self.run_update_analytics_counts(options)
        finally:
            os.rmdir(settings.ANALYTICS_LOCK_DIR)
    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
        # installation_epoch relies on there being at least one realm; we
        # shouldn't run the analytics code if that condition isn't satisfied
        if not Realm.objects.exists():
            logger.info("No realms, stopping update_analytics_counts")
            return
        fill_to_time = parse_datetime(options['time'])
        if options['utc']:
            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
        if fill_to_time.tzinfo is None:
            raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
        if options['stat'] is not None:
            stats = [COUNT_STATS[options['stat']]]
        else:
            stats = list(COUNT_STATS.values())
        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
        if options['verbose']:
            start = time.time()
            last = start
        for stat in stats:
            process_count_stat(stat, fill_to_time)
            if options['verbose']:
                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
                last = time.time()
        if options['verbose']:
            print("Finished updating analytics counts through %s in %.3fs" %
                  (fill_to_time, time.time() - start))
        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
            send_analytics_to_remote_server()
--- a/analytics/management/commands/user_stats.py
+++ b/analytics/management/commands/user_stats.py
@@ -1,42 +1,37 @@
 from __future__ import absolute_import
 import datetime
-from argparse import ArgumentParser
+import pytz
 from typing import Any
 from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
+from zerver.models import UserProfile, Realm, Stream, Message
 from zerver.models import Message, Realm, Stream, UserProfile, get_realm
 class Command(BaseCommand):
    help = "Generate statistics on user activity."
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def messages_sent_by(self, user, week):
-        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
-                            help="realm to generate statistics for")
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
    def messages_sent_by(self, user: UserProfile, week: int) -> int:
        start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
        end = timezone_now() - datetime.timedelta(days=week*7)
        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
-        if options['realms']:
+        if args:
            try:
-                realms = [get_realm(string_id) for string_id in options['realms']]
+                realms = [Realm.objects.get(domain=domain) for domain in args]
-            except Realm.DoesNotExist as e:
+            except Realm.DoesNotExist, e:
-                print(e)
+                print e
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
-            print(realm.string_id)
+            print realm.domain
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
-            print("%d users" % (len(user_profiles),))
+            print "%d users" % (len(user_profiles),)
-            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
+            print "%d streams" % (len(Stream.objects.filter(realm=realm)),)
            for user_profile in user_profiles:
-                print("%35s" % (user_profile.email,), end=' ')
+                print "%35s" % (user_profile.email,),
                for week in range(10):
-                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
+                    print "%5d" % (self.messages_sent_by(user_profile, week)),
-                print("")
+                print ""
--- a/analytics/migrations/0001_initial.py
+++ b/analytics/migrations/0001_initial.py
@@ -1,110 +0,0 @@
 # -*- coding: utf-8 -*-
 import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('zerver', '0030_realm_org_type'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]
    operations = [
        migrations.CreateModel(
            name='Anomaly',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('info', models.CharField(max_length=1000)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='HuddleCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('huddle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='InstallationCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='RealmCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='StreamCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
                ('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream')),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='UserCount',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                ('property', models.CharField(max_length=40)),
                ('end_time', models.DateTimeField()),
                ('interval', models.CharField(max_length=20)),
                ('value', models.BigIntegerField()),
                ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
            ],
            bases=(models.Model,),
        ),
        migrations.AlterUniqueTogether(
            name='usercount',
            unique_together=set([('user', 'property', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='streamcount',
            unique_together=set([('stream', 'property', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='realmcount',
            unique_together=set([('realm', 'property', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='installationcount',
            unique_together=set([('property', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='huddlecount',
            unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
        ),
    ]
--- a/analytics/migrations/0002_remove_huddlecount.py
+++ b/analytics/migrations/0002_remove_huddlecount.py
@@ -1,30 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0001_initial'),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name='huddlecount',
            unique_together=set([]),
        ),
        migrations.RemoveField(
            model_name='huddlecount',
            name='anomaly',
        ),
        migrations.RemoveField(
            model_name='huddlecount',
            name='huddle',
        ),
        migrations.RemoveField(
            model_name='huddlecount',
            name='user',
        ),
        migrations.DeleteModel(
            name='HuddleCount',
        ),
    ]
--- a/analytics/migrations/0003_fillstate.py
+++ b/analytics/migrations/0003_fillstate.py
@@ -1,22 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0002_remove_huddlecount'),
    ]
    operations = [
        migrations.CreateModel(
            name='FillState',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('property', models.CharField(unique=True, max_length=40)),
                ('end_time', models.DateTimeField()),
                ('state', models.PositiveSmallIntegerField()),
                ('last_modified', models.DateTimeField(auto_now=True)),
            ],
            bases=(models.Model,),
        ),
    ]
--- a/analytics/migrations/0004_add_subgroup.py
+++ b/analytics/migrations/0004_add_subgroup.py
@@ -1,31 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0003_fillstate'),
    ]
    operations = [
        migrations.AddField(
            model_name='installationcount',
            name='subgroup',
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name='realmcount',
            name='subgroup',
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name='streamcount',
            name='subgroup',
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name='usercount',
            name='subgroup',
            field=models.CharField(max_length=16, null=True),
        ),
    ]
--- a/analytics/migrations/0005_alter_field_size.py
+++ b/analytics/migrations/0005_alter_field_size.py
@@ -1,51 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0004_add_subgroup'),
    ]
    operations = [
        migrations.AlterField(
            model_name='installationcount',
            name='interval',
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name='installationcount',
            name='property',
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name='realmcount',
            name='interval',
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name='realmcount',
            name='property',
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name='streamcount',
            name='interval',
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name='streamcount',
            name='property',
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name='usercount',
            name='interval',
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name='usercount',
            name='property',
            field=models.CharField(max_length=32),
        ),
    ]
--- a/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
+++ b/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
@@ -1,27 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0005_alter_field_size'),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name='installationcount',
            unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='realmcount',
            unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='streamcount',
            unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
        ),
        migrations.AlterUniqueTogether(
            name='usercount',
            unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
        ),
    ]
--- a/analytics/migrations/0007_remove_interval.py
+++ b/analytics/migrations/0007_remove_interval.py
@@ -1,44 +0,0 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-16 20:50
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0006_add_subgroup_to_unique_constraints'),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name='installationcount',
            unique_together=set([('property', 'subgroup', 'end_time')]),
        ),
        migrations.RemoveField(
            model_name='installationcount',
            name='interval',
        ),
        migrations.AlterUniqueTogether(
            name='realmcount',
            unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
        ),
        migrations.RemoveField(
            model_name='realmcount',
            name='interval',
        ),
        migrations.AlterUniqueTogether(
            name='streamcount',
            unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
        ),
        migrations.RemoveField(
            model_name='streamcount',
            name='interval',
        ),
        migrations.AlterUniqueTogether(
            name='usercount',
            unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
        ),
        migrations.RemoveField(
            model_name='usercount',
            name='interval',
        ),
    ]
--- a/analytics/migrations/0008_add_count_indexes.py
+++ b/analytics/migrations/0008_add_count_indexes.py
@@ -1,25 +0,0 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-02-01 22:28
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ('zerver', '0050_userprofile_avatar_version'),
        ('analytics', '0007_remove_interval'),
    ]
    operations = [
        migrations.AlterIndexTogether(
            name='realmcount',
            index_together=set([('property', 'end_time')]),
        ),
        migrations.AlterIndexTogether(
            name='streamcount',
            index_together=set([('property', 'realm', 'end_time')]),
        ),
        migrations.AlterIndexTogether(
            name='usercount',
            index_together=set([('property', 'realm', 'end_time')]),
        ),
    ]
--- a/analytics/migrations/0009_remove_messages_to_stream_stat.py
+++ b/analytics/migrations/0009_remove_messages_to_stream_stat.py
@@ -1,28 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations
 from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
    UserCount = apps.get_model('analytics', 'UserCount')
    StreamCount = apps.get_model('analytics', 'StreamCount')
    RealmCount = apps.get_model('analytics', 'RealmCount')
    InstallationCount = apps.get_model('analytics', 'InstallationCount')
    FillState = apps.get_model('analytics', 'FillState')
    property = 'messages_sent_to_stream:is_bot'
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0008_add_count_indexes'),
    ]
    operations = [
        migrations.RunPython(delete_messages_sent_to_stream_stat),
    ]
--- a/analytics/migrations/0010_clear_messages_sent_values.py
+++ b/analytics/migrations/0010_clear_messages_sent_values.py
@@ -1,26 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations
 from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
    UserCount = apps.get_model('analytics', 'UserCount')
    StreamCount = apps.get_model('analytics', 'StreamCount')
    RealmCount = apps.get_model('analytics', 'RealmCount')
    InstallationCount = apps.get_model('analytics', 'InstallationCount')
    FillState = apps.get_model('analytics', 'FillState')
    property = 'messages_sent:message_type:day'
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 class Migration(migrations.Migration):
    dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
    operations = [
        migrations.RunPython(clear_message_sent_by_message_type_values),
    ]
--- a/analytics/migrations/0011_clear_analytics_tables.py
+++ b/analytics/migrations/0011_clear_analytics_tables.py
@@ -1,27 +0,0 @@
 # -*- coding: utf-8 -*-
 from django.db import migrations
 from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
    UserCount = apps.get_model('analytics', 'UserCount')
    StreamCount = apps.get_model('analytics', 'StreamCount')
    RealmCount = apps.get_model('analytics', 'RealmCount')
    InstallationCount = apps.get_model('analytics', 'InstallationCount')
    FillState = apps.get_model('analytics', 'FillState')
    UserCount.objects.all().delete()
    StreamCount.objects.all().delete()
    RealmCount.objects.all().delete()
    InstallationCount.objects.all().delete()
    FillState.objects.all().delete()
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0010_clear_messages_sent_values'),
    ]
    operations = [
        migrations.RunPython(clear_analytics_tables),
    ]
--- a/analytics/migrations/0012_add_on_delete.py
+++ b/analytics/migrations/0012_add_on_delete.py
@@ -1,36 +0,0 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.6 on 2018-01-29 08:14
 from __future__ import unicode_literals
 from django.db import migrations, models
 import django.db.models.deletion
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0011_clear_analytics_tables'),
    ]
    operations = [
        migrations.AlterField(
            model_name='installationcount',
            name='anomaly',
            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
        ),
        migrations.AlterField(
            model_name='realmcount',
            name='anomaly',
            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
        ),
        migrations.AlterField(
            model_name='streamcount',
            name='anomaly',
            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
        ),
        migrations.AlterField(
            model_name='usercount',
            name='anomaly',
            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
        ),
    ]
--- a/analytics/migrations/0013_remove_anomaly.py
+++ b/analytics/migrations/0013_remove_anomaly.py
@@ -1,34 +0,0 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.18 on 2019-02-02 02:47
 from __future__ import unicode_literals
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ('analytics', '0012_add_on_delete'),
    ]
    operations = [
        migrations.RemoveField(
            model_name='installationcount',
            name='anomaly',
        ),
        migrations.RemoveField(
            model_name='realmcount',
            name='anomaly',
        ),
        migrations.RemoveField(
            model_name='streamcount',
            name='anomaly',
        ),
        migrations.RemoveField(
            model_name='usercount',
            name='anomaly',
        ),
        migrations.DeleteModel(
            name='Anomaly',
        ),
    ]
--- a/analytics/models.py
+++ b/analytics/models.py
@@ -1,92 +0,0 @@
 import datetime
 from typing import Optional
 from django.db import models
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Stream, UserProfile
 class FillState(models.Model):
    property = models.CharField(max_length=40, unique=True)  # type: str
    end_time = models.DateTimeField()  # type: datetime.datetime
    # Valid states are {DONE, STARTED}
    DONE = 1
    STARTED = 2
    state = models.PositiveSmallIntegerField()  # type: int
    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
    def __str__(self) -> str:
        return "<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
 def installation_epoch() -> datetime.datetime:
    earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
    return floor_to_day(earliest_realm_creation)
 def last_successful_fill(property: str) -> Optional[datetime.datetime]:
    fillstate = FillState.objects.filter(property=property).first()
    if fillstate is None:
        return None
    if fillstate.state == FillState.DONE:
        return fillstate.end_time
    return fillstate.end_time - datetime.timedelta(hours=1)
 class BaseCount(models.Model):
    # Note: When inheriting from BaseCount, you may want to rearrange
    # the order of the columns in the migration to make sure they
    # match how you'd like the table to be arranged.
    property = models.CharField(max_length=32)  # type: str
    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[str]
    end_time = models.DateTimeField()  # type: datetime.datetime
    value = models.BigIntegerField()  # type: int
    class Meta:
        abstract = True
 class InstallationCount(BaseCount):
    class Meta:
        unique_together = ("property", "subgroup", "end_time")
    def __str__(self) -> str:
        return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
 class RealmCount(BaseCount):
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        unique_together = ("realm", "property", "subgroup", "end_time")
        index_together = ["property", "end_time"]
    def __str__(self) -> str:
        return "<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
 class UserCount(BaseCount):
    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        unique_together = ("user", "property", "subgroup", "end_time")
        # This index dramatically improves the performance of
        # aggregating from users to realms
        index_together = ["property", "realm", "end_time"]
    def __str__(self) -> str:
        return "<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
 class StreamCount(BaseCount):
    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        unique_together = ("stream", "property", "subgroup", "end_time")
        # This index dramatically improves the performance of
        # aggregating from streams to realms
        index_together = ["property", "realm", "end_time"]
    def __str__(self) -> str:
        return "<StreamCount: %s %s %s %s %s>" % (
            self.stream, self.property, self.subgroup, self.value, self.id)
--- a/analytics/tests/test_counts.py
+++ b/analytics/tests/test_counts.py
--- a/analytics/tests/test_fixtures.py
+++ b/analytics/tests/test_fixtures.py
@@ -1,29 +0,0 @@
 from analytics.lib.counts import CountStat
 from analytics.lib.fixtures import generate_time_series_data
 from zerver.lib.test_classes import ZulipTestCase
 # A very light test suite; the code being tested is not run in production.
 class TestFixtures(ZulipTestCase):
    def test_deterministic_settings(self) -> None:
        # test basic business_hour / non_business_hour calculation
        # test we get an array of the right length with frequency=CountStat.DAY
        data = generate_time_series_data(
            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
        data = generate_time_series_data(
            days=1, business_hours_base=2000, non_business_hours_base=1500,
            growth=2, spikiness=0, frequency=CountStat.HOUR)
        # test we get an array of the right length with frequency=CountStat.HOUR
        self.assertEqual(len(data), 24)
        # test that growth doesn't affect the first data point
        self.assertEqual(data[0], 2000)
        # test that the last data point is growth times what it otherwise would be
        self.assertEqual(data[-1], 1500*2)
        # test autocorrelation == 1, since that's the easiest value to test
        data = generate_time_series_data(
            days=1, business_hours_base=2000, non_business_hours_base=2000,
            autocorrelation=1, frequency=CountStat.HOUR)
        self.assertEqual(data[0], data[1])
        self.assertEqual(data[0], data[-1])
--- a/analytics/tests/test_views.py
+++ b/analytics/tests/test_views.py
@@ -1,404 +0,0 @@
 from datetime import datetime, timedelta
 from typing import List, Optional
 import mock
 from django.utils.timezone import utc
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
 from analytics.models import FillState, \
    RealmCount, UserCount, last_successful_fill
 from analytics.views import rewrite_client_arrays, \
    sort_by_totals, sort_client_labels
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.timestamp import ceiling_to_day, \
    ceiling_to_hour, datetime_to_timestamp
 from zerver.models import Client, get_realm
 class TestStatsEndpoint(ZulipTestCase):
    def test_stats(self) -> None:
        self.user = self.example_user('hamlet')
        self.login(self.user.email)
        result = self.client_get('/stats')
        self.assertEqual(result.status_code, 200)
        # Check that we get something back
        self.assert_in_response("Zulip analytics for", result)
    def test_guest_user_cant_access_stats(self) -> None:
        self.user = self.example_user('polonius')
        self.login(self.user.email)
        result = self.client_get('/stats')
        self.assert_json_error(result, "Not allowed for guest users", 400)
        result = self.client_get('/json/analytics/chart_data')
        self.assert_json_error(result, "Not allowed for guest users", 400)
    def test_stats_for_realm(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login(user_profile.email)
        result = self.client_get('/stats/realm/zulip/')
        self.assertEqual(result.status_code, 302)
        user_profile = self.example_user('hamlet')
        user_profile.is_staff = True
        user_profile.save(update_fields=['is_staff'])
        result = self.client_get('/stats/realm/not_existing_realm/')
        self.assertEqual(result.status_code, 302)
        result = self.client_get('/stats/realm/zulip/')
        self.assertEqual(result.status_code, 200)
        self.assert_in_response("Zulip analytics for", result)
    def test_stats_for_installation(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login(user_profile.email)
        result = self.client_get('/stats/installation')
        self.assertEqual(result.status_code, 302)
        user_profile = self.example_user('hamlet')
        user_profile.is_staff = True
        user_profile.save(update_fields=['is_staff'])
        result = self.client_get('/stats/installation')
        self.assertEqual(result.status_code, 200)
        self.assert_in_response("Zulip analytics for", result)
 class TestGetChartData(ZulipTestCase):
    def setUp(self) -> None:
        self.realm = get_realm('zulip')
        self.user = self.example_user('hamlet')
        self.login(self.user.email)
        self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
                               for i in range(4)]
        self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
                              for i in range(4)]
    def data(self, i: int) -> List[int]:
        return [0, 0, i, 0]
    def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]],
                    user_subgroups: List[str]) -> None:
        if stat.frequency == CountStat.HOUR:
            insert_time = self.end_times_hour[2]
            fill_time = self.end_times_hour[-1]
        if stat.frequency == CountStat.DAY:
            insert_time = self.end_times_day[2]
            fill_time = self.end_times_day[-1]
        RealmCount.objects.bulk_create([
            RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
                       value=100+i, realm=self.realm)
            for i, subgroup in enumerate(realm_subgroups)])
        UserCount.objects.bulk_create([
            UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
                      value=200+i, realm=self.realm, user=self.user)
            for i, subgroup in enumerate(user_subgroups)])
        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
    def test_number_of_humans(self) -> None:
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['1day_actives::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['active_users_audit:is_bot:day']
        self.insert_data(stat, ['false'], [])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data, {
            'msg': '',
            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
            'frequency': CountStat.DAY,
            'everyone': {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)},
            'display_order': None,
            'result': 'success',
        })
    def test_messages_sent_over_time(self) -> None:
        stat = COUNT_STATS['messages_sent:is_bot:hour']
        self.insert_data(stat, ['true', 'false'], ['false'])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data, {
            'msg': '',
            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
            'frequency': CountStat.HOUR,
            'everyone': {'bot': self.data(100), 'human': self.data(101)},
            'user': {'bot': self.data(0), 'human': self.data(200)},
            'display_order': None,
            'result': 'success',
        })
    def test_messages_sent_by_message_type(self) -> None:
        stat = COUNT_STATS['messages_sent:message_type:day']
        self.insert_data(stat, ['public_stream', 'private_message'],
                         ['public_stream', 'private_stream'])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_by_message_type'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data, {
            'msg': '',
            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
            'frequency': CountStat.DAY,
            'everyone': {'Public streams': self.data(100), 'Private streams': self.data(0),
                         'Private messages': self.data(101), 'Group private messages': self.data(0)},
            'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
                     'Private messages': self.data(0), 'Group private messages': self.data(0)},
            'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
            'result': 'success',
        })
    def test_messages_sent_by_client(self) -> None:
        stat = COUNT_STATS['messages_sent:client:day']
        client1 = Client.objects.create(name='client 1')
        client2 = Client.objects.create(name='client 2')
        client3 = Client.objects.create(name='client 3')
        client4 = Client.objects.create(name='client 4')
        self.insert_data(stat, [client4.id, client3.id, client2.id],
                         [client3.id, client1.id])
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_by_client'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data, {
            'msg': '',
            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
            'frequency': CountStat.DAY,
            'everyone': {'client 4': self.data(100), 'client 3': self.data(101),
                         'client 2': self.data(102)},
            'user': {'client 3': self.data(200), 'client 1': self.data(201)},
            'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
            'result': 'success',
        })
    def test_include_empty_subgroups(self) -> None:
        FillState.objects.create(
            property='realm_active_humans::day', end_time=self.end_times_day[0],
            state=FillState.DONE)
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['everyone'], {"_1day": [0], "_15day": [0], "all_time": [0]})
        self.assertFalse('user' in data)
        FillState.objects.create(
            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0],
            state=FillState.DONE)
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_over_time'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['everyone'], {'human': [0], 'bot': [0]})
        self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
        FillState.objects.create(
            property='messages_sent:message_type:day', end_time=self.end_times_day[0],
            state=FillState.DONE)
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_by_message_type'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['everyone'], {
            'Public streams': [0], 'Private streams': [0],
            'Private messages': [0], 'Group private messages': [0]})
        self.assertEqual(data['user'], {
            'Public streams': [0], 'Private streams': [0],
            'Private messages': [0], 'Group private messages': [0]})
        FillState.objects.create(
            property='messages_sent:client:day', end_time=self.end_times_day[0],
            state=FillState.DONE)
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'messages_sent_by_client'})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['everyone'], {})
        self.assertEqual(data['user'], {})
    def test_start_and_end(self) -> None:
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['1day_actives::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['active_users_audit:is_bot:day']
        self.insert_data(stat, ['false'], [])
        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
        # valid start and end
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'start': end_time_timestamps[1],
                                  'end': end_time_timestamps[2]})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['end_times'], end_time_timestamps[1:3])
        self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})
        # start later then end
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'start': end_time_timestamps[2],
                                  'end': end_time_timestamps[1]})
        self.assert_json_error_contains(result, 'Start time is later than')
    def test_min_length(self) -> None:
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['1day_actives::day']
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS['active_users_audit:is_bot:day']
        self.insert_data(stat, ['false'], [])
        # test min_length is too short to change anything
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'min_length': 2})
        self.assert_json_success(result)
        data = result.json()
        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
        self.assertEqual(data['everyone'], {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)})
        # test min_length larger than filled data
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'number_of_humans',
                                  'min_length': 5})
        self.assert_json_success(result)
        data = result.json()
        end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
        self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
        self.assertEqual(data['everyone'], {'_1day': [0]+self.data(100), '_15day': [0]+self.data(100), 'all_time': [0]+self.data(100)})
    def test_non_existent_chart(self) -> None:
        result = self.client_get('/json/analytics/chart_data',
                                 {'chart_name': 'does_not_exist'})
        self.assert_json_error_contains(result, 'Unknown chart name')
    def test_analytics_not_running(self) -> None:
        # try to get data for a valid chart, but before we've put anything in the database
        # (e.g. before update_analytics_counts has been run)
        with mock.patch('logging.warning'):
            result = self.client_get('/json/analytics/chart_data',
                                     {'chart_name': 'number_of_humans'})
        self.assert_json_error_contains(result, 'No analytics data available')
    def test_get_chart_data_for_realm(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login(user_profile.email)
        result = self.client_get('/json/analytics/chart_data/realm/zulip/',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_error(result, "Must be an server administrator", 400)
        user_profile = self.example_user('hamlet')
        user_profile.is_staff = True
        user_profile.save(update_fields=['is_staff'])
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_error(result, 'Invalid organization', 400)
        result = self.client_get('/json/analytics/chart_data/realm/zulip',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_success(result)
    def test_get_chart_data_for_installation(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login(user_profile.email)
        result = self.client_get('/json/analytics/chart_data/installation',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_error(result, "Must be an server administrator", 400)
        user_profile = self.example_user('hamlet')
        user_profile.is_staff = True
        user_profile.save(update_fields=['is_staff'])
        stat = COUNT_STATS['realm_active_humans::day']
        self.insert_data(stat, [None], [])
        result = self.client_get('/json/analytics/chart_data/installation',
                                 {'chart_name': 'number_of_humans'})
        self.assert_json_success(result)
 class TestGetChartDataHelpers(ZulipTestCase):
    # last_successful_fill is in analytics/models.py, but get_chart_data is
    # the only function that uses it at the moment
    def test_last_successful_fill(self) -> None:
        self.assertIsNone(last_successful_fill('non-existant'))
        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
        fillstate = FillState.objects.create(property='property', end_time=a_time,
                                             state=FillState.DONE)
        self.assertEqual(last_successful_fill('property'), a_time)
        fillstate.state = FillState.STARTED
        fillstate.save()
        self.assertEqual(last_successful_fill('property'), one_hour_before)
    def test_sort_by_totals(self) -> None:
        empty = []  # type: List[int]
        value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
        self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
    def test_sort_client_labels(self) -> None:
        data = {'everyone': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
                'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
        self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
 class TestTimeRange(ZulipTestCase):
    def test_time_range(self) -> None:
        HOUR = timedelta(hours=1)
        DAY = timedelta(days=1)
        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
        floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
        # test start == end
        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
        # test start == end == boundary, and min_length == 0
        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
        # test start and end on different boundaries
        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, None),
                         [floor_hour, floor_hour+HOUR])
        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, None),
                         [floor_day, floor_day+DAY])
        # test min_length
        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, 4),
                         [floor_hour-2*HOUR, floor_hour-HOUR, floor_hour, floor_hour+HOUR])
        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, 4),
                         [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
 class TestMapArrays(ZulipTestCase):
    def test_map_arrays(self) -> None:
        a = {'desktop app 1.0': [1, 2, 3],
             'desktop app 2.0': [10, 12, 13],
             'desktop app 3.0': [21, 22, 23],
             'website': [1, 2, 3],
             'ZulipiOS': [1, 2, 3],
             'ZulipElectron': [2, 5, 7],
             'ZulipMobile': [1, 5, 7],
             'ZulipPython': [1, 2, 3],
             'API: Python': [1, 2, 3],
             'SomethingRandom': [4, 5, 6],
             'ZulipGitHubWebhook': [7, 7, 9],
             'ZulipAndroid': [64, 63, 65]}
        result = rewrite_client_arrays(a)
        self.assertEqual(result,
                         {'Old desktop app': [32, 36, 39],
                          'Old iOS app': [1, 2, 3],
                          'Desktop app': [2, 5, 7],
                          'Mobile app': [1, 5, 7],
                          'Website': [1, 2, 3],
                          'Python API': [2, 4, 6],
                          'SomethingRandom': [4, 5, 6],
                          'GitHub webhook': [7, 7, 9],
                          'Old Android app': [64, 63, 65]})
--- a/analytics/urls.py
+++ b/analytics/urls.py
@@ -1,59 +1,8 @@
-from django.conf.urls import include, url
+from django.conf.urls import patterns, url
-import analytics.views
+urlpatterns = patterns('analytics.views',
-from zerver.lib.rest import rest_dispatch
+    url(r'^activity$', 'get_activity'),
    url(r'^realm_activity/(?P<realm>[\S]+)/$', 'get_realm_activity'),
    url(r'^user_activity/(?P<email>[\S]+)/$', 'get_user_activity'),
 )
 i18n_urlpatterns = [
    # Server admin (user_profile.is_staff) visible stats pages
    url(r'^activity$', analytics.views.get_activity,
        name='analytics.views.get_activity'),
    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
        name='analytics.views.get_realm_activity'),
    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
        name='analytics.views.get_user_activity'),
    url(r'^stats/realm/(?P<realm_str>[\S]+)/$', analytics.views.stats_for_realm,
        name='analytics.views.stats_for_realm'),
    url(r'^stats/installation$', analytics.views.stats_for_installation,
        name='analytics.views.stats_for_installation'),
    url(r'^stats/remote/(?P<remote_server_id>[\S]+)/installation$',
        analytics.views.stats_for_remote_installation,
        name='analytics.views.stats_for_remote_installation'),
    url(r'^stats/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)/$',
        analytics.views.stats_for_remote_realm,
        name='analytics.views.stats_for_remote_realm'),
    # User-visible stats page
    url(r'^stats$', analytics.views.stats,
        name='analytics.views.stats'),
 ]
 # These endpoints are a part of the API (V1), which uses:
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
 # * Takes and returns json-formatted data
 #
 # See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
 #
 # All of these paths are accessed by either a /json or /api prefix
 v1_api_and_json_patterns = [
    # get data for the graphs at /stats
    url(r'^analytics/chart_data$', rest_dispatch,
        {'GET': 'analytics.views.get_chart_data'}),
    url(r'^analytics/chart_data/realm/(?P<realm_str>[\S]+)$', rest_dispatch,
        {'GET': 'analytics.views.get_chart_data_for_realm'}),
    url(r'^analytics/chart_data/installation$', rest_dispatch,
        {'GET': 'analytics.views.get_chart_data_for_installation'}),
    url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/installation$', rest_dispatch,
        {'GET': 'analytics.views.get_chart_data_for_remote_installation'}),
    url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)$',
        rest_dispatch,
        {'GET': 'analytics.views.get_chart_data_for_remote_realm'}),
 ]
 i18n_urlpatterns += [
    url(r'^api/v1/', include(v1_api_and_json_patterns)),
    url(r'^json/', include(v1_api_and_json_patterns)),
 ]
 urlpatterns = i18n_urlpatterns
--- a/analytics/views.py
+++ b/analytics/views.py
--- a/api/MANIFEST.in
+++ b/api/MANIFEST.in
@@ -0,0 +1,10 @@
 recursive-include integrations *
 include README.md
 include examples/zuliprc
 include examples/send-message
 include examples/subscribe
 include examples/get-public-streams
 include examples/unsubscribe
 include examples/list-members
 include examples/list-subscriptions
 include examples/print-messages
--- a/api/README.md
+++ b/api/README.md
@@ -0,0 +1,106 @@
 #### Dependencies
 The [Zulip API](https://zulip.com/api) Python bindings require the
 following Python libraries:
 * simplejson
 * requests (version >= 0.12.1)
 #### Installing
 This package uses distutils, so you can just run:
    python setup.py install
 #### Using the API
 For now, the only fully supported API operation is sending a message.
 The other API queries work, but are under active development, so
 please make sure we know you're using them so that we can notify you
 as we make any changes to them.
 The easiest way to use these API bindings is to base your tools off
 of the example tools under examples/ in this distribution.
 If you place your API key in the config file `~/.zuliprc` the Python
 API bindings will automatically read it in. The format of the config
 file is as follows:
    [api]
    key=<api key from the web interface>
    email=<your email address>
 If you are using Zulip Enterprise, you should also add
    site=<your Zulip Enterprise server's URI>
 Alternatively, you may explicitly use "--user" and "--api-key" in our
 examples, which is especially useful if you are running several bots
 which share a home directory.  There is also a "--site" option for
 setting the Zulip Enterprise server on the command line.
 You can obtain your Zulip API key, create bots, and manage bots all
 from your Zulip [settings page](https://zulip.com/#settings).
 A typical simple bot sending API messages will look as follows:
 At the top of the file:
    # Make sure the Zulip API distribution's root directory is in sys.path, then:
    import zulip
    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
 When you want to send a message:
    message = {
      "type": "stream",
      "to": ["support"],
      "subject": "your subject",
      "content": "your content",
    }
    zulip_client.send_message(message)
 Additional examples:
    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
                         'subject': 'feedback', 'to': ['support']})
    client.send_message({'type': 'private', 'content': 'Zulip rules!',
                         'to': ['user1@example.com', 'user2@example.com']})
 send_message() returns a dict guaranteed to contain the following
 keys: msg, result.  For successful calls, result will be "success" and
 msg will be the empty string.  On error, result will be "error" and
 msg will describe what went wrong.
 #### Logging
 The Zulip API comes with a ZulipStream class which can be used with the
 logging module:
 ```
 import zulip
 import logging
 stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
 logger = logging.getLogger("your_logger")
 logger.addHandler(logging.StreamHandler(stream))
 logger.setLevel(logging.DEBUG)
 logger.info("This is an INFO test.")
 logger.debug("This is a DEBUG test.")
 logger.warn("This is a WARN test.")
 logger.error("This is a ERROR test.")
 ```
 #### Sending messages
 You can use the included `zulip-send` script to send messages via the
 API directly from existing scripts.
    zulip-send hamlet@example.com cordelia@example.com -m \
        "Conscience doth make cowards of us all."
 Alternatively, if you don't want to use your ~/.zuliprc file:
    zulip-send --user shakespeare-bot@example.com \
        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
        hamlet@example.com cordelia@example.com -m \
        "Conscience doth make cowards of us all."
--- a/api/bin/zulip-send
+++ b/api/bin/zulip-send
@@ -0,0 +1,126 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # zulip-send -- Sends a message to the specified recipients.
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 import logging
 logging.basicConfig()
 log = logging.getLogger('zulip-send')
 def do_send_message(client, message_data ):
    '''Sends a message and optionally prints status about the same.'''
    if message_data['type'] == 'stream':
        log.info('Sending message to stream "%s", subject "%s"... ' % \
            (message_data['to'], message_data['subject']))
    else:
        log.info('Sending message to %s... ' % message_data['to'])
    response = client.send_message(message_data)
    if response['result'] == 'success':
        log.info('Message sent.')
        return True
    else:
        log.error(response['msg'])
        return False
 def main(argv=None):
    if argv is None:
        argv = sys.argv
    usage = """%prog [options] [recipient...]
    Sends a message specified recipients.
    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
    '--user' and '--api-key' arguments.
    """
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
    import zulip
    parser = optparse.OptionParser(usage=usage)
    # Grab parser options from the API common set
    parser.add_option_group(zulip.generate_option_group(parser))
    parser.add_option('-m', '--message',
                      help='Specifies the message to send, prevents interactive prompting.')
    group = optparse.OptionGroup(parser, 'Stream parameters')
    group.add_option('-s', '--stream',
                      dest='stream',
                      action='store',
                      help='Allows the user to specify a stream for the message.')
    group.add_option('-S', '--subject',
                      dest='subject',
                      action='store',
                      help='Allows the user to specify a subject for the message.')
    parser.add_option_group(group)
    (options, recipients) = parser.parse_args(argv[1:])
    if options.verbose:
        logging.getLogger().setLevel(logging.INFO)
    # Sanity check user data
    if len(recipients) != 0 and (options.stream or options.subject):
        parser.error('You cannot specify both a username and a stream/subject.')
    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
        parser.error('Stream messages must have a subject')
    if len(recipients) == 0 and not (options.stream and options.subject):
        parser.error('You must specify a stream/subject or at least one recipient.')
    client = zulip.init_from_options(options)
    if not options.message:
        options.message = sys.stdin.read()
    if options.stream:
        message_data = {
            'type': 'stream',
            'content': options.message,
            'subject': options.subject,
            'to': options.stream,
        }
    else:
        message_data = {
            'type': 'private',
            'content': options.message,
            'to': recipients,
        }
    if not do_send_message(client, message_data):
        return 1
 if __name__ == '__main__':
    sys.exit(main())
--- a/api/examples/create-user
+++ b/api/examples/create-user
@@ -0,0 +1,54 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 from os import path
 import optparse
 usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
 Create a user. You must be a realm admin to use this API, and the user
 will be created in your realm.
 Example: create-user --site=http://localhost:9991 --user=rwbarton@zulip.com --new-email=jarthur@zulip.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
 """
 sys.path.append(path.join(path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--new-email')
 parser.add_option('--new-password')
 parser.add_option('--new-full-name')
 parser.add_option('--new-short-name')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print client.create_user({
        'email': options.new_email,
        'password': options.new_password,
        'full_name': options.new_full_name,
        'short_name': options.new_short_name
        })
--- a/api/examples/edit-message
+++ b/api/examples/edit-message
@@ -0,0 +1,56 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
 Edits a message that you sent
 Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option('--message-id', default="")
 parser.add_option('--subject', default="")
 parser.add_option('--content',   default="")
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 message_data = {
    "message_id": options.message_id,
 }
 if options.subject != "":
    message_data["subject"] = options.subject
 if options.content != "":
    message_data["content"] = options.content
 print client.update_message(message_data)
--- a/api/examples/get-public-streams
+++ b/api/examples/get-public-streams
@@ -0,0 +1,46 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out all the public streams in the realm.
 Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print client.get_streams(include_public=True, include_subscribed=False)
--- a/api/examples/list-members
+++ b/api/examples/list-members
@@ -0,0 +1,45 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
 List the names and e-mail addresses of the people in your realm.
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 for user in client.get_members()["members"]:
    print user["full_name"], user["email"]
--- a/api/examples/list-subscriptions
+++ b/api/examples/list-subscriptions
@@ -0,0 +1,45 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out a list of the user's subscriptions.
 Example: list-subscriptions --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print client.list_subscriptions()
--- a/api/examples/print-messages
+++ b/api/examples/print-messages
@@ -0,0 +1,49 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out each message received by the indicated bot or user.
 Example: print-messages --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 def print_message(message):
    print message
 # This is a blocking call, and will continuously poll for new messages
 client.call_on_each_message(print_message)
--- a/api/examples/print-next-message
+++ b/api/examples/print-next-message
@@ -0,0 +1,45 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """print-next-message --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out the next message received by the user.
 Example: print-next-messages --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print client.get_messages({})
--- a/api/examples/send-message
+++ b/api/examples/send-message
@@ -0,0 +1,57 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
 Sends a test message to the specified recipients.
 Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
 Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 parser = optparse.OptionParser(usage=usage)
 parser.add_option('--subject', default="test")
 parser.add_option('--message', default="test message")
 parser.add_option('--type',   default='private')
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 if len(args) == 0:
    parser.error("You must specify recipients")
 client = zulip.init_from_options(options)
 message_data = {
    "type": options.type,
    "content": options.message,
    "subject": options.subject,
    "to": args,
 }
 print client.send_message(message_data)
--- a/api/examples/subscribe
+++ b/api/examples/subscribe
@@ -0,0 +1,52 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
 Ensures the user is subscribed to the listed streams.
 Examples: subscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
          subscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--streams', default='')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 if options.streams == "":
    print >>sys.stderr, "Usage:", parser.usage
    sys.exit(1)
 print client.add_subscriptions([{"name": stream_name} for stream_name in
                                options.streams.split()])
--- a/api/examples/unsubscribe
+++ b/api/examples/unsubscribe
@@ -0,0 +1,51 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
 Ensures the user is not subscribed to the listed streams.
 Examples: unsubscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
          unsubscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--streams', default='')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 if options.streams == "":
    print >>sys.stderr, "Usage:", parser.usage
    sys.exit(1)
 print client.remove_subscriptions(options.streams.split())
--- a/api/examples/zuliprc
+++ b/api/examples/zuliprc
@@ -0,0 +1,4 @@
 ; Save this file as ~/.zuliprc
 [api]
 key=<your bot's api key from the web interface>
 email=<your bot's email address>
--- a/api/integrations/asana/zulip_asana_config.py
+++ b/api/integrations/asana/zulip_asana_config.py
@@ -0,0 +1,57 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 ### REQUIRED CONFIGURATION ###
 # Change these values to your Asana credentials.
 ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
 # Change these values to the credentials for your Asana bot.
 ZULIP_USER = "asana-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # The Zulip stream that will receive Asana task updates.
 ZULIP_STREAM_NAME = "asana"
 ### OPTIONAL CONFIGURATION ###
 # Set to None for logging to stdout when testing, and to a file for
 # logging when deployed.
 #LOG_FILE = "/var/tmp/zulip_asana.log"
 LOG_FILE = None
 # This file is used to resume this mirror in case the script shuts down.
 # It is required and needs to be writeable.
 RESUME_FILE = "/var/tmp/zulip_asana.state"
 # When initially started, how many hours of messages to include.
 ASANA_INITIAL_HISTORY_HOURS = 1
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
 # If properly installed, the Zulip API should be in your import
 # path, but if not, set a custom path below
 ZULIP_API_PATH = None
--- a/api/integrations/asana/zulip_asana_mirror
+++ b/api/integrations/asana/zulip_asana_mirror
@@ -0,0 +1,277 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Asana integration for Zulip
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import base64
 from datetime import datetime, timedelta
 import dateutil.parser
 import dateutil.tz
 import json
 import logging
 import os
 import time
 import urllib2
 import sys
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_asana_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 if config.LOG_FILE:
    logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
 else:
    logging.basicConfig(level=logging.INFO)
 client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
                      site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
 def fetch_from_asana(path):
    """
    Request a resource through the Asana API, authenticating using
    HTTP basic auth.
    """
    auth = base64.encodestring('%s:' % (config.ASANA_API_KEY,))
    headers = {"Authorization": "Basic %s" % auth}
    url = "https://app.asana.com/api/1.0" + path
    request = urllib2.Request(url, None, headers)
    result = urllib2.urlopen(request)
    return json.load(result)
 def send_zulip(topic, content):
    """
    Send a message to Zulip using the configured stream and bot credentials.
    """
    message = {"type": "stream",
               "sender": config.ZULIP_USER,
               "to": config.ZULIP_STREAM_NAME,
               "subject": topic,
               "content": content,
               }
    return client.send_message(message)
 def datestring_to_datetime(datestring):
    """
    Given an ISO 8601 datestring, return the corresponding datetime object.
    """
    return dateutil.parser.parse(datestring).replace(
        tzinfo=dateutil.tz.gettz('Z'))
 class TaskDict(dict):
    """
    A helper class to turn a dictionary with task information into an
    object where each of the keys is an attribute for easy access.
    """
    def __getattr__(self, field):
        return self.get(field)
 def format_topic(task, projects):
    """
    Return a string that will be the Zulip message topic for this task.
    """
    # Tasks can be associated with multiple projects, but in practice they seem
    # to mostly be associated with one.
    project_name = projects[task.projects[0]["id"]]
    return "%s: %s" % (project_name, task.name)
 def format_assignee(task, users):
    """
    Return a string describing the task's assignee.
    """
    if task.assignee:
        assignee_name = users[task.assignee["id"]]
        assignee_info = "**Assigned to**: %s (%s)" % (
            assignee_name, task.assignee_status)
    else:
        assignee_info = "**Status**: Unassigned"
    return assignee_info
 def format_due_date(task):
    """
    Return a string describing the task's due date.
    """
    if task.due_on:
        due_date_info = "**Due on**: %s" % (task.due_on,)
    else:
        due_date_info = "**Due date**: None"
    return due_date_info
 def format_task_creation_event(task, projects, users):
    """
    Format the topic and content for a newly-created task.
    """
    topic = format_topic(task, projects)
    assignee_info = format_assignee(task, users)
    due_date_info = format_due_date(task)
    content = """Task **%s** created:
 ~~~ quote
 %s
 ~~~
 %s
 %s
 """ % (task.name, task.notes, assignee_info, due_date_info)
    return topic, content
 def format_task_completion_event(task, projects, users):
    """
    Format the topic and content for a completed task.
    """
    topic = format_topic(task, projects)
    assignee_info = format_assignee(task, users)
    due_date_info = format_due_date(task)
    content = """Task **%s** completed. :white_check_mark:
 %s
 %s
 """ % (task.name, assignee_info, due_date_info)
    return topic, content
 def since():
    """
    Return a newness threshold for task events to be processed.
    """
    # If we have a record of the last event processed and it is recent, use it,
    # else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
    def default_since():
        return datetime.utcnow() - timedelta(
            hours=config.ASANA_INITIAL_HISTORY_HOURS)
    if os.path.exists(config.RESUME_FILE):
        try:
            with open(config.RESUME_FILE, "r") as f:
                datestring = f.readline().strip()
                timestamp = float(datestring)
                max_timestamp_processed = datetime.fromtimestamp(timestamp)
                logging.info("Reading from resume file: " + datestring)
        except (ValueError,IOError) as e:
            logging.warn("Could not open resume file: %s" % (
                    e.message or e.strerror,))
            max_timestamp_processed = default_since()
    else:
        logging.info("No resume file, processing an initial history.")
        max_timestamp_processed = default_since()
    # Even if we can read a timestamp from RESUME_FILE, if it is old don't use
    # it.
    return max(max_timestamp_processed, default_since())
 def process_new_events():
    """
    Forward new Asana task events to Zulip.
    """
    # In task queries, Asana only exposes IDs for projects and users, so we need
    # to look up the mappings.
    projects = dict((elt["id"], elt["name"]) for elt in \
                        fetch_from_asana("/projects")["data"])
    users = dict((elt["id"], elt["name"]) for elt in \
                     fetch_from_asana("/users")["data"])
    cutoff = since()
    max_timestamp_processed = cutoff
    time_operations = (("created_at", format_task_creation_event),
                       ("completed_at", format_task_completion_event))
    task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
                   "modified_at", "due_on", "name", "notes", "projects"]
    # First, gather all of the tasks that need processing. We'll
    # process them in order.
    new_events = []
    for project_id in projects:
        project_url = "/projects/%d/tasks?opt_fields=%s" % (
            project_id, ",".join(task_fields))
        tasks = fetch_from_asana(project_url)["data"]
        for task in tasks:
            task = TaskDict(task)
            for time_field, operation in time_operations:
                if task[time_field]:
                    operation_time = datestring_to_datetime(task[time_field])
                    if operation_time > cutoff:
                        new_events.append((operation_time, time_field, operation, task))
    new_events.sort()
    now = datetime.utcnow()
    for operation_time, time_field, operation, task in new_events:
        # Unfortunately, creating an Asana task is not an atomic operation. If
        # the task was just created, or is missing basic information, it is
        # probably because the task is still being filled out -- wait until the
        # next round to process it.
        if (time_field == "created_at") and \
                (now - operation_time < timedelta(seconds=30)):
            # The task was just created, give the user some time to fill out
            # more information.
            return
        if (time_field == "created_at") and (not task.name) and \
                (now - operation_time < timedelta(seconds=60)):
            # If this new task hasn't had a name for a full 30 seconds, assume
            # you don't plan on giving it one.
            return
        topic, content = operation(task, projects, users)
        logging.info("Sending Zulip for " + topic)
        result = send_zulip(topic, content)
        # If the Zulip wasn't sent successfully, don't update the
        # max timestamp processed so the task has another change to
        # be forwarded. Exit, giving temporary issues time to
        # resolve.
        if not result.get("result"):
            logging.warn("Malformed result, exiting:")
            logging.warn(result)
            return
        if result["result"] != "success":
            logging.warn(result["msg"])
            return
        if operation_time > max_timestamp_processed:
            max_timestamp_processed = operation_time
    if max_timestamp_processed > cutoff:
        max_datestring = max_timestamp_processed.strftime("%s.%f")
        logging.info("Updating resume file: " + max_datestring)
        open(config.RESUME_FILE, 'w').write(max_datestring)
 while True:
    try:
        process_new_events()
        time.sleep(5)
    except KeyboardInterrupt:
        logging.info("Shutting down...")
        logging.info("Set LOG_FILE to log to a file instead of stdout.")
        break
--- a/api/integrations/basecamp/zulip_basecamp_config.py
+++ b/api/integrations/basecamp/zulip_basecamp_config.py
@@ -0,0 +1,53 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Change these values to configure authentication for basecamp account
 BASECAMP_ACCOUNT_ID = "12345678"
 BASECAMP_USERNAME = "foo@example.com"
 BASECAMP_PASSWORD = "p455w0rd"
 # This script will mirror this many hours of history on the first run.
 # On subsequent runs this value is ignored.
 BASECAMP_INITIAL_HISTORY_HOURS = 0
 # Change these values to configure Zulip authentication for the plugin
 ZULIP_USER = "basecamp-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 ZULIP_STREAM_NAME = "basecamp"
 ## If properly installed, the Zulip API should be in your import
 ## path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
 # If you wish to log to a file rather than stdout/stderr,
 # please fill this out your desired path
 LOG_FILE = None
 # This file is used to resume this mirror in case the script shuts down.
 # It is required and needs to be writeable.
 RESUME_FILE = "/var/tmp/zulip_basecamp.state"
--- a/api/integrations/basecamp/zulip_basecamp_mirror
+++ b/api/integrations/basecamp/zulip_basecamp_mirror
@@ -0,0 +1,180 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Zulip mirror of Basecamp activity
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
 # The "basecamp-mirror.py" script is run continuously, possibly on a work computer
 # or preferably on a server.
 # You may need to install the python-requests library.
 import requests
 import logging
 import time
 import re
 import sys
 import os
 from datetime import datetime, timedelta
 from HTMLParser import HTMLParser
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_basecamp_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipBasecamp/" + VERSION)
 user_agent = "Basecamp To Zulip Mirroring script (support@zulip.com)"
 htmlParser = HTMLParser()
 # find some form of JSON loader/dumper, with a preference order for speed.
 json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
 while len(json_implementations):
    try:
        json = __import__(json_implementations.pop(0))
        break
    except ImportError:
        continue
 # void function that checks the permissions of the files this script needs.
 def check_permissions():
    # check that the log file can be written
    if config.LOG_FILE:
        try:
            open(config.LOG_FILE, "w")
        except IOError as e:
            sys.stderr("Could not open up log for writing:")
            sys.stderr(e)
    # check that the resume file can be written (this creates if it doesn't exist)
    try:
        open(config.RESUME_FILE, "a+")
    except IOError as e:
        sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
        sys.stderr(e)
 # builds the message dict for sending a message with the Zulip API
 def build_message(event):
    if not (event.has_key('bucket') and event.has_key('creator') and event.has_key('html_url')):
        logging.error("Perhaps the Basecamp API changed behavior? "
                      "This event doesn't have the expected format:\n%s" %(event,))
        return None
    # adjust the topic length to be bounded to 60 characters
    topic = event['bucket']['name']
    if len(topic) > 60:
        topic = topic[0:57] + "..."
    # get the action and target values
    action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
    target = htmlParser.unescape(event.get('target', ''))
    # Some events have "excerpts", which we blockquote
    excerpt = htmlParser.unescape(event.get('excerpt',''))
    if excerpt.strip() == "":
        message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
    else:
        message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
    # assemble the message data dict
    message_data = {
        "type": "stream",
        "to": config.ZULIP_STREAM_NAME,
        "subject": topic,
        "content": message,
    }
    return message_data
 # the main run loop for this mirror script
 def run_mirror():
    # we should have the right (write) permissions on the resume file, as seen
    # in check_permissions, but it may still be empty or corrupted
    try:
        with open(config.RESUME_FILE) as f:
            since = f.read()
        since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
        assert since, "resume file does not meet expected format"
        since = since.string
    except (AssertionError,IOError) as e:
        logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
        since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
    try:
        # we use an exponential backoff approach when we get 429 (Too Many Requests).
        sleepInterval = 1
        while 1:
            time.sleep(sleepInterval)
            response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
                                    params={'since': since},
                                    auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
                                    headers = {"User-Agent": user_agent})
            if response.status_code == 200:
                sleepInterval = 1
                events = json.loads(response.text)
                if len(events):
                    logging.info("Got event(s): %s" % (response.text,))
            if response.status_code >= 500:
                logging.error(response.status_code)
                continue
            if response.status_code == 429:
                # exponential backoff
                sleepInterval *= 2
                logging.error(response.status_code)
                continue
            if response.status_code == 400:
                logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
                sys.exit(-1)
            if response.status_code == 401:
                logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
                sys.exit(-1)
            if len(events):
                since = events[0]['created_at']
            for event in reversed(events):
                message_data = build_message(event)
                if not message_data:
                    continue
                zulip_api_result = client.send_message(message_data)
                if zulip_api_result['result'] == "success":
                    logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
                else:
                    logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
                # update 'since' each time in case we get KeyboardInterrupted
                since = event['created_at']
                # avoid hitting rate-limit
                time.sleep(0.2)
    except KeyboardInterrupt:
        logging.info("Shutting down, please hold")
        open("events.last", 'w').write(since)
        logging.info("Done!")
 if __name__ == "__main__":
    if not isinstance(config.RESUME_FILE, basestring):
        sys.stderr("RESUME_FILE path not given; refusing to continue")
    check_permissions()
    if config.LOG_FILE:
        logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
    else:
        logging.basicConfig(level=logging.INFO)
    run_mirror()
--- a/api/integrations/codebase/zulip_codebase_config.py
+++ b/api/integrations/codebase/zulip_codebase_config.py
@@ -0,0 +1,62 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Change these values to configure authentication for your codebase account
 # Note that this is the Codebase API Username, found in the Settings page
 # for your account
 CODEBASE_API_USERNAME = "foo@example.com"
 CODEBASE_API_KEY = "1234561234567abcdef"
 # The URL of your codebase setup
 CODEBASE_ROOT_URL = "https://YOUR_COMPANY.codebasehq.com"
 # When initially started, how many hours of messages to include.
 # Note that the Codebase API only returns the 20 latest events,
 # if you have more than 20 events that fit within this window,
 # earlier ones may be lost
 CODEBASE_INITIAL_HISTORY_HOURS = 12
 # Change these values to configure Zulip authentication for the plugin
 ZULIP_USER = "codebase-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # The streams to send commit information and ticket information to
 ZULIP_COMMITS_STREAM_NAME = "codebase"
 ZULIP_TICKETS_STREAM_NAME = "tickets"
 # If properly installed, the Zulip API should be in your import
 # path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
 # If you wish to log to a file rather than stdout/stderr,
 # please fill this out your desired path
 LOG_FILE = None
 # This file is used to resume this mirror in case the script shuts down.
 # It is required and needs to be writeable.
 RESUME_FILE = "/var/tmp/zulip_codebase.state"
--- a/api/integrations/codebase/zulip_codebase_mirror
+++ b/api/integrations/codebase/zulip_codebase_mirror
@@ -0,0 +1,318 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Zulip mirror of Codebase HQ activity
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
 # The "codebase-mirror.py" script is run continuously, possibly on a work computer
 # or preferably on a server.
 #
 # When restarted, it will attempt to pick up where it left off.
 #
 # You may need to install the python-requests library, as well as python-dateutil
 import requests
 import logging
 import time
 import sys
 import os
 from datetime import datetime, timedelta
 import dateutil.parser
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_codebase_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipCodebase/" + VERSION)
 user_agent = "Codebase To Zulip Mirroring script (support@zulip.com)"
 # find some form of JSON loader/dumper, with a preference order for speed.
 json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
 while len(json_implementations):
    try:
        json = __import__(json_implementations.pop(0))
        break
    except ImportError:
        continue
 def make_api_call(path):
    response = requests.get("https://api3.codebasehq.com/%s" % (path,),
                             auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
                             params={'raw': True},
                             headers = {"User-Agent": user_agent,
                                        "Content-Type": "application/json",
                                        "Accept": "application/json"})
    if response.status_code == 200:
        return json.loads(response.text)
    if response.status_code >= 500:
        logging.error(response.status_code)
        return None
    if response.status_code == 403:
        logging.error("Bad authorization from Codebase. Please check your credentials")
        sys.exit(-1)
    else:
        logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
        return None
 def make_url(path):
    return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
 def handle_event(event):
    event = event['event']
    event_type = event['type']
    actor_name = event['actor_name']
    raw_props = event.get('raw_properties', {})
    project_link = raw_props.get('project_permalink')
    subject = None
    content = None
    if event_type == 'repository_creation':
        stream = config.ZULIP_COMMITS_STREAM_NAME
        project_name = raw_props.get('name')
        project_repo_type = raw_props.get('scm_type')
        url = make_url("projects/%s" % project_link)
        scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
        subject = "Repository %s Created" % (project_name,)
        content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
    elif event_type == 'push':
        stream = config.ZULIP_COMMITS_STREAM_NAME
        num_commits = raw_props.get('commits_count')
        branch = raw_props.get('ref_name')
        project = raw_props.get('project_name')
        repo_link = raw_props.get('repository_permalink')
        deleted_ref = raw_props.get('deleted_ref')
        new_ref = raw_props.get('new_ref')
        subject = "Push to %s on %s" % (branch, project)
        if deleted_ref:
            content = "%s deleted branch %s from %s" % (actor_name, branch, project)
        else:
            if new_ref:
                branch = "new branch %s" % (branch, )
            content = "%s pushed %s commit(s) to %s in project %s:\n\n" % \
                        (actor_name, num_commits, branch, project)
            for commit in raw_props.get('commits'):
                ref = commit.get('ref')
                url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
                message = commit.get('message')
                content += "* [%s](%s): %s\n" % (ref, url, message)
    elif event_type == 'ticketing_ticket':
        stream = config.ZULIP_TICKETS_STREAM_NAME
        num = raw_props.get('number')
        name = raw_props.get('subject')
        assignee = raw_props.get('assignee')
        priority = raw_props.get('priority')
        url = make_url("projects/%s/tickets/%s" % (project_link, num))
        if assignee is None:
            assignee = "no one"
        subject = "#%s: %s" % (num, name)
        content = """%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" % \
                    (actor_name, num, url, priority, assignee, name)
    elif event_type == 'ticketing_note':
        stream = config.ZULIP_TICKETS_STREAM_NAME
        num = raw_props.get('number')
        name = raw_props.get('subject')
        body = raw_props.get('content')
        changes = raw_props.get('changes')
        url = make_url("projects/%s/tickets/%s" % (project_link, num))
        subject = "#%s: %s" % (num, name)
        content = ""
        if body is not None and len(body) > 0:
            content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
        if 'status_id' in changes:
            status_change = changes.get('status_id')
            content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
    elif event_type == 'ticketing_milestone':
        stream = config.ZULIP_TICKETS_STREAM_NAME
        name = raw_props.get('name')
        identifier = raw_props.get('identifier')
        url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
        subject = name
        content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
    elif event_type == 'comment':
        stream = config.ZULIP_COMMITS_STREAM_NAME
        comment = raw_props.get('content')
        commit = raw_props.get('commit_ref')
        # If there's a commit id, it's a comment to a commit
        if commit:
            repo_link = raw_props.get('repository_permalink')
            url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
            subject = "%s commented on %s" % (actor_name, commit)
            content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
        else:
            # Otherwise, this is a Discussion item, and handle it
            subj = raw_props.get("subject")
            category = raw_props.get("category")
            comment_content = raw_props.get("content")
            subject = "Discussion: %s" % (subj,)
            if category:
                content = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~" % (actor_name, category, comment_content)
            else:
                content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
    elif event_type == 'deployment':
        stream = config.ZULIP_COMMITS_STREAM_NAME
        start_ref = raw_props.get('start_ref')
        end_ref = raw_props.get('end_ref')
        environment = raw_props.get('environment')
        servers = raw_props.get('servers')
        repo_link = raw_props.get('repository_permalink')
        start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
        end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
        between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (project_link, repo_link, start_ref, end_ref))
        subject = "Deployment to %s" % (environment,)
        content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % \
                    (actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment)
        if servers is not None:
            content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
    elif event_type == 'named_tree':
        # Docs say named_tree type used for new/deleting branches and tags,
        # but experimental testing showed that they were all sent as 'push' events
        pass
    elif event_type == 'wiki_page':
        logging.warn("Wiki page notifications not yet implemented")
    elif event_type == 'sprint_creation':
        logging.warn("Sprint notifications not yet implemented")
    elif event_type == 'sprint_ended':
        logging.warn("Sprint notifications not yet implemented")
    else:
        logging.info("Unknown event type %s, ignoring!" % (event_type,))
    if subject and content:
        if len(subject) > 60:
            subject = subject[:57].rstrip() + '...'
        res = client.send_message({"type": "stream",
                                   "to": stream,
                                   "subject": subject,
                                   "content": content})
        if res['result'] == 'success':
            logging.info("Successfully sent Zulip with id: %s" % (res['id']))
        else:
            logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
 # the main run loop for this mirror script
 def run_mirror():
    # we should have the right (write) permissions on the resume file, as seen
    # in check_permissions, but it may still be empty or corrupted
    def default_since():
        return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
    try:
        with open(config.RESUME_FILE) as f:
            timestamp = f.read()
        if timestamp == '':
            since = default_since()
        else:
            timestamp = int(timestamp, 10)
            since = datetime.fromtimestamp(timestamp)
    except (ValueError,IOError) as e:
        logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
        since = default_since()
    try:
        sleepInterval = 1
        while 1:
            events = make_api_call("activity")[::-1]
            if events is not None:
                sleepInterval = 1
                for event in events:
                    timestamp = event.get('event', {}).get('timestamp', '')
                    event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None)
                    if event_date > since:
                        handle_event(event)
                        since = event_date
            else:
                # back off a bit
                if sleepInterval < 22:
                    sleepInterval += 4
            time.sleep(sleepInterval)
    except KeyboardInterrupt:
        open(config.RESUME_FILE, 'w').write(since.strftime("%s"));
        logging.info("Shutting down Codebase mirror")
 # void function that checks the permissions of the files this script needs.
 def check_permissions():
    # check that the log file can be written
    if config.LOG_FILE:
        try:
            open(config.LOG_FILE, "w")
        except IOError as e:
            sys.stderr("Could not open up log for writing:")
            sys.stderr(e)
    # check that the resume file can be written (this creates if it doesn't exist)
    try:
        open(config.RESUME_FILE, "a+")
    except IOError as e:
        sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
        sys.stderr(e)
 if __name__ == "__main__":
    if not isinstance(config.RESUME_FILE, basestring):
        sys.stderr("RESUME_FILE path not given; refusing to continue")
    check_permissions()
    if config.LOG_FILE:
        logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
    else:
        logging.basicConfig(level=logging.WARNING)
    run_mirror()
--- a/api/integrations/git/post-receive
+++ b/api/integrations/git/post-receive
@@ -0,0 +1,130 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Zulip notification post-receive hook.
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
 # The "post-receive" script is run after receive-pack has accepted a pack
 # and the repository has been updated.  It is passed arguments in through
 # stdin in the form
 #  <oldrev> <newrev> <refname>
 # For example:
 #  aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
 import os
 import sys
 import subprocess
 import os.path
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_git_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipGit/" + VERSION)
 # check_output is backported from subprocess.py in Python 2.7
 def check_output(*popenargs, **kwargs):
    if 'stdout' in kwargs:
        raise ValueError('stdout argument not allowed, it will be overridden.')
    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
    output, unused_err = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = kwargs.get("args")
        if cmd is None:
            cmd = popenargs[0]
        raise subprocess.CalledProcessError(retcode, cmd, output=output)
    return output
 subprocess.check_output = check_output
 def git_repository_name():
    output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
    if output.strip() == "true":
        return os.path.basename(os.getcwd())[:-len(".git")]
    else:
        return os.path.basename(os.path.dirname(os.getcwd()))
 def git_commit_range(oldrev, newrev):
    log_cmd = ["git", "log", "--reverse",
               "--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
    commits = ''
    for ln in subprocess.check_output(log_cmd).splitlines():
        author_email, commit_id, subject  = ln.split(None, 2)
        if hasattr(config, "format_commit_message"):
            commits += config.format_commit_message(author_email, subject, commit_id)
        else:
            commits += '!avatar(%s) %s\n' % (author_email, subject)
    return commits
 def send_bot_message(oldrev, newrev, refname):
    repo_name   = git_repository_name()
    branch      = refname.replace('refs/heads/', '')
    destination = config.commit_notice_destination(repo_name, branch, newrev)
    if destination is None:
        # Don't forward the notice anywhere
        return
    new_head = newrev[:12]
    old_head = oldrev[:12]
    if (oldrev == '0000000000000000000000000000000000000000' or
        newrev == '0000000000000000000000000000000000000000'):
        # New branch pushed or old branch removed
        added = ''
        removed = ''
    else:
        added   = git_commit_range(oldrev, newrev)
        removed = git_commit_range(newrev, oldrev)
    if oldrev == '0000000000000000000000000000000000000000':
        message = '`%s` was pushed to new branch `%s`' % (new_head, branch)
    elif newrev == '0000000000000000000000000000000000000000':
        message = 'branch `%s` was removed (was `%s`)' % (branch, old_head)
    elif removed:
        message = '`%s` was pushed to `%s`, **REMOVING**:\n\n%s' % (new_head, branch, removed)
        if added:
            message += '\n**and adding**:\n\n' + added
        message += '\n**A HISTORY REWRITE HAS OCCURRED!**'
        message += '\n@everyone: Please check your local branches to deal with this.'
    elif added:
        message = '`%s` was deployed to `%s` with:\n\n%s' % (new_head, branch, added)
    else:
        message = '`%s` was pushed to `%s`... but nothing changed?' % (new_head, branch)
    message_data = {
        "type": "stream",
        "to": destination["stream"],
        "subject": destination["subject"],
        "content": message,
    }
    client.send_message(message_data)
 for ln in sys.stdin:
    oldrev, newrev, refname = ln.strip().split()
    send_bot_message(oldrev, newrev, refname)
--- a/api/integrations/git/zulip_git_config.py
+++ b/api/integrations/git/zulip_git_config.py
@@ -0,0 +1,65 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Change these values to configure authentication for the plugin
 ZULIP_USER = "git-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # commit_notice_destination() lets you customize where commit notices
 # are sent to with the full power of a Python function.
 #
 # It takes the following arguments:
 # * repo   = the name of the git repository
 # * branch = the name of the branch that was pushed to
 # * commit = the commit id
 #
 # Returns a dictionary encoding the stream and subject to send the
 # notification to (or None to send no notification).
 #
 # The default code below will send every commit pushed to "master" to
 # * stream "commits"
 # * topic "master"
 # And similarly for branch "test-post-receive" (for use when testing).
 def commit_notice_destination(repo, branch, commit):
    if branch in ["master", "test-post-receive"]:
        return dict(stream  = "commits",
                    subject = u"%s" % (branch,))
    # Return None for cases where you don't want a notice sent
    return None
 # Modify this function to change how commits are displayed; the most
 # common customization is to include a link to the commit in your
 # graphical repository viewer, e.g.
 #
 # return '!avatar(%s) [%s](https://example.com/commits/%s)\n' % (author, subject, commit_id)
 def format_commit_message(author, subject, commit_id):
    return '!avatar(%s) %s\n' % (author, subject)
 ## If properly installed, the Zulip API should be in your import
 ## path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/hg/zulip-changegroup.py
+++ b/api/integrations/hg/zulip-changegroup.py
@@ -0,0 +1,170 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Zulip hook for Mercurial changeset pushes.
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
 #
 # This hook is called when changesets are pushed to the master repository (ie
 # `hg push`). See https://zulip.com/integrations for installation instructions.
 import zulip
 VERSION = "0.9"
 def format_summary_line(web_url, user, base, tip, branch, node):
    """
    Format the first line of the message, which contains summary
    information about the changeset and links to the changelog if a
    web URL has been configured:
    Jane Doe <jane@example.com> pushed 1 commit to master (170:e494a5be3393):
    """
    revcount = tip - base
    plural = "s" if revcount > 1 else ""
    if web_url:
        shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
        summary_url = "{shortlog}{tip}?revcount={revcount}".format(
            shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
        formatted_commit_count = "[{revcount} commit{s}]({url})".format(
            revcount=revcount, s=plural, url=summary_url)
    else:
        formatted_commit_count = "{revcount} commit{s}".format(
            revcount=revcount, s=plural)
    return u"**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
        user=user, commits=formatted_commit_count, branch=branch, tip=tip,
        node=node[:12])
 def format_commit_lines(web_url, repo, base, tip):
    """
    Format the per-commit information for the message, including the one-line
    commit summary and a link to the diff if a web URL has been configured:
    """
    if web_url:
        rev_base_url = web_url.rstrip("/") + "/rev/"
    commit_summaries = []
    for rev in range(base, tip):
        rev_node = repo.changelog.node(rev)
        rev_ctx = repo.changectx(rev_node)
        one_liner = rev_ctx.description().split("\n")[0]
        if web_url:
            summary_url = rev_base_url + str(rev_ctx)
            summary = "* [{summary}]({url})".format(
                summary=one_liner, url=summary_url)
        else:
            summary = "* {summary}".format(summary=one_liner)
        commit_summaries.append(summary)
    return "\n".join(summary for summary in commit_summaries)
 def send_zulip(email, api_key, site, stream, subject, content):
    """
    Send a message to Zulip using the provided credentials, which should be for
    a bot in most cases.
    """
    client = zulip.Client(email=email, api_key=api_key,
                          site=site,
                          client="ZulipMercurial/" + VERSION)
    message_data = {
        "type": "stream",
        "to": stream,
        "subject": subject,
        "content": content,
    }
    client.send_message(message_data)
 def get_config(ui, item):
    try:
        # configlist returns everything in lists.
        return ui.configlist('zulip', item)[0]
    except IndexError:
        return None
 def hook(ui, repo, **kwargs):
    """
    Invoked by configuring a [hook] entry in .hg/hgrc.
    """
    hooktype = kwargs["hooktype"]
    node = kwargs["node"]
    ui.debug("Zulip: received {hooktype} event\n".format(hooktype=hooktype))
    if hooktype != "changegroup":
        ui.warn("Zulip: {hooktype} not supported\n".format(hooktype=hooktype))
        exit(1)
    ctx = repo.changectx(node)
    branch = ctx.branch()
    # If `branches` isn't specified, notify on all branches.
    branch_whitelist = get_config(ui, "branches")
    branch_blacklist = get_config(ui, "ignore_branches")
    if branch_whitelist:
        # Only send notifications on branches we are watching.
        watched_branches = [b.lower().strip() for b in branch_whitelist.split(",")]
        if branch.lower() not in watched_branches:
            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
            exit(0)
    if branch_blacklist:
        # Don't send notifications for branches we've ignored.
        ignored_branches = [b.lower().strip() for b in branch_blacklist.split(",")]
        if branch.lower() in ignored_branches:
            ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
            exit(0)
    # The first and final commits in the changeset.
    base = repo[node].rev()
    tip = len(repo)
    email = get_config(ui, "email")
    api_key = get_config(ui, "api_key")
    site = get_config(ui, "site")
    if not (email and api_key):
        ui.warn("Zulip: missing email or api_key configurations\n")
        ui.warn("in the [zulip] section of your .hg/hgrc.\n")
        exit(1)
    stream = get_config(ui, "stream")
    # Give a default stream if one isn't provided.
    if not stream:
        stream = "commits"
    web_url = get_config(ui, "web_url")
    user = ctx.user()
    content = format_summary_line(web_url, user, base, tip, branch, node)
    content += format_commit_lines(web_url, repo, base, tip)
    subject = branch
    ui.debug("Sending to Zulip:\n")
    ui.debug(content + "\n")
    send_zulip(email, api_key, site, stream, subject, content)
--- a/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
+++ b/api/integrations/jira/org/humbug/jira/ZulipListener.groovy
@@ -0,0 +1,149 @@
 /*
 * Copyright (c) 2014 Zulip, Inc
 */
 package org.zulip.jira
 import static com.atlassian.jira.event.type.EventType.*
 import com.atlassian.jira.event.issue.AbstractIssueEventListener
 import com.atlassian.jira.event.issue.IssueEvent
 import java.util.logging.Level
 import java.util.logging.Logger
 import org.apache.commons.httpclient.HttpClient
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.methods.PostMethod
 import org.apache.commons.httpclient.NameValuePair
 class ZulipListener extends AbstractIssueEventListener {
    Logger LOGGER = Logger.getLogger(ZulipListener.class.getName());
    // The email address of one of the bots you created on your Zulip settings page.
    String zulipEmail = ""
    // That bot's API key.
    String zulipAPIKey = ""
    // What stream to send messages to. Must already exist.
    String zulipStream = "jira"
    // The base JIRA url for browsing
    String issueBaseUrl = "https://jira.COMPANY.com/browse/"
    // Your zulip domain, only change if you have a custom one
    String base_url = "https://api.zulip.com"
    @Override
    void workflowEvent(IssueEvent event) {
      processIssueEvent(event)
    }
    String processIssueEvent(IssueEvent event) {
      String author = event.user.displayName
      String issueId = event.issue.key
      String issueUrl = issueBaseUrl + issueId
      String issueUrlMd = String.format("[%s](%s)", issueId, issueBaseUrl + issueId)
      String title = event.issue.summary
      String subject = truncate(String.format("%s: %s", issueId, title), 60)
      String assignee = "no one"
      if (event.issue.assignee) {
        assignee = event.issue.assignee.name
      }
      String comment = "";
      if (event.comment) {
        comment = event.comment.body
      }
      String content;
      // Event types:
      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/event/type/EventType.html
      // Issue API:
      // https://docs.atlassian.com/jira/5.0/com/atlassian/jira/issue/Issue.html
      switch (event.getEventTypeId()) {
        case ISSUE_COMMENTED_ID:
          content = String.format("%s **updated** %s with comment:\n\n> %s",
                                  author, issueUrlMd, comment)
          break
        case ISSUE_CREATED_ID:
          content = String.format("%s **created** %s priority %s, assigned to **%s**: \n\n> %s",
                                  author, issueUrlMd, event.issue.priorityObject.name,
                                  assignee, title)
          break
        case ISSUE_ASSIGNED_ID:
          content = String.format("%s **reassigned** %s to **%s**",
                                  author, issueUrlMd, assignee)
          break
        case ISSUE_DELETED_ID:
          content = String.format("%s **deleted** %s!",
                                  author, issueUrlMd)
          break
        case ISSUE_RESOLVED_ID:
          content = String.format("%s **resolved** %s as %s:\n\n> %s",
                                  author, issueUrlMd, event.issue.resolutionObject.name,
                                  comment)
          break
        case ISSUE_CLOSED_ID:
          content = String.format("%s **closed** %s with resolution %s:\n\n> %s",
                                  author, issueUrlMd, event.issue.resolutionObject.name,
                                  comment)
          break
        case ISSUE_REOPENED_ID:
          content = String.format("%s **reopened** %s:\n\n> %s",
                                  author, issueUrlMd, comment)
          break
        default:
          return
      }
      sendStreamMessage(zulipStream, subject, content)
    }
    String post(String method, NameValuePair[] parameters) {
      PostMethod post = new PostMethod(zulipUrl(method))
      post.setRequestHeader("Content-Type", post.FORM_URL_ENCODED_CONTENT_TYPE)
      // TODO: Include more useful data in the User-agent
      post.setRequestHeader("User-agent", "ZulipJira/0.1")
      try {
        post.setRequestBody(parameters)
        HttpClient client = new HttpClient()
        client.executeMethod(post)
        String response = post.getResponseBodyAsString()
        if (post.getStatusCode() != HttpStatus.SC_OK) {
          String params = ""
          for (NameValuePair pair: parameters) {
              params += "\n" + pair.getName() + ":" + pair.getValue()
          }
          LOGGER.log(Level.SEVERE, "Error sending Zulip message:\n" + response + "\n\n" +
                                   "We sent:" + params)
        }
        return response;
      } catch (IOException e) {
        throw new RuntimeException(e)
      } finally {
        post.releaseConnection()
      }
    }
    String truncate(String string, int length) {
      if (string.length() < length) {
        return string
      }
      return string.substring(0, length - 3) + "..."
    }
    String sendStreamMessage(String stream, String subject, String message) {
      NameValuePair[] body = [new NameValuePair("api-key", zulipAPIKey),
                              new NameValuePair("email",   zulipEmail),
                              new NameValuePair("type",    "stream"),
                              new NameValuePair("to",      stream),
                              new NameValuePair("subject", subject),
                              new NameValuePair("content", message)]
      return post("send_message", body);
    }
    String zulipUrl(method) {
      return base_url + "/v1/" + method
    }
 }
--- a/api/integrations/nagios/nagios-notify-zulip
+++ b/api/integrations/nagios/nagios-notify-zulip
@@ -0,0 +1,49 @@
 #!/usr/bin/env python
 import optparse
 import zulip
 VERSION = "0.9"
 # Nagios passes the notification details as command line options.
 # In Nagios, "output" means "first line of output", and "long
 # output" means "other lines of output".
 parser = optparse.OptionParser()
 parser.add_option('--output',      default='')
 parser.add_option('--long-output', default='')
 parser.add_option('--stream', default='nagios')
 parser.add_option('--config', default='/etc/nagios3/zuliprc')
 for opt in ('type', 'host', 'service', 'state'):
    parser.add_option('--' + opt)
 (opts, args) = parser.parse_args()
 client = zulip.Client(config_file=opts.config, client="ZulipNagios/" + VERSION)
 msg = dict(type='stream', to=opts.stream)
 # Set a subject based on the host or service in question.  This enables
 # threaded discussion of multiple concurrent issues, and provides useful
 # context when narrowed.
 #
 # We send PROBLEM and RECOVERY messages to the same subject.
 if opts.service is None:
    # Host notification
    thing = 'host'
    msg['subject'] = 'host %s' % (opts.host,)
 else:
    # Service notification
    thing = 'service'
    msg['subject'] = 'service %s on %s' % (opts.service, opts.host)
 if len(msg['subject']) > 60:
    msg['subject'] = msg['subject'][0:57].rstrip() + "..."
 # e.g. **PROBLEM**: service is CRITICAL
 msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
 # The "long output" can contain newlines represented by "\n" escape sequences.
 # The Nagios mail command uses /usr/bin/printf "%b" to expand these.
 # We will be more conservative and handle just this one escape sequence.
 output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip()
 if output:
    # Put any command output in a code block.
    msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
 client.send_message(msg)
--- a/api/integrations/nagios/zulip_nagios.cfg
+++ b/api/integrations/nagios/zulip_nagios.cfg
@@ -0,0 +1,21 @@
 define contact{
        contact_name                    zulip
        alias                           zulip
        service_notification_period     24x7
        host_notification_period        24x7
        service_notification_options    w,u,c,r
        host_notification_options       d,r
        service_notification_commands   notify-service-by-zulip
        host_notification_commands      notify-host-by-zulip
        }
 # Zulip commands
 define command {
    command_name    notify-host-by-zulip
    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
 }
 define command {
    command_name    notify-service-by-zulip
    command_line    /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
 }
--- a/api/integrations/nagios/zuliprc.example
+++ b/api/integrations/nagios/zuliprc.example
@@ -0,0 +1,5 @@
 # Fill these values in with the appropriate values for your realm, and
 # then install this value at /etc/nagios3/zuliprc
 [api]
 email = nagios-bot@example.com
 key = 0123456789abcdef0123456789abcdef
--- a/api/integrations/perforce/git_p4.py
+++ b/api/integrations/perforce/git_p4.py
--- a/api/integrations/perforce/license.txt
+++ b/api/integrations/perforce/license.txt
@@ -0,0 +1,26 @@
 git_p4.py was downloaded from https://raw.github.com/git/git/34022ba/git-p4.py
 The header of that file references <http://opensource.org/licenses/mit-license.php>,
 the textual contents of which are included below.
 The MIT License (MIT)
 Copyright (c) <year> <copyright holders>
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
--- a/api/integrations/perforce/zulip_change-commit.py
+++ b/api/integrations/perforce/zulip_change-commit.py
@@ -0,0 +1,88 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 '''Zulip notification change-commit hook.
 In Perforce, The "change-commit" trigger is fired after a metadata has been
 created, files have been transferred, and the changelist comitted to the depot
 database.
 This specific trigger expects command-line arguments in the form:
  %change% %changeroot%
 For example:
  1234 //depot/security/src/
 '''
 import os
 import sys
 import os.path
 import git_p4
 __version__ = "0.1"
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_perforce_config as config
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipPerforce/" + __version__)
 try:
    changelist = int(sys.argv[1])
    changeroot = sys.argv[2]
 except IndexError:
    print >> sys.stderr, "Wrong number of arguments.\n\n",
    print >> sys.stderr,  __doc__
    sys.exit(-1)
 except ValueError:
    print >> sys.stderr, "First argument must be an integer.\n\n",
    print >> sys.stderr, __doc__
    sys.exit(-1)
 metadata = git_p4.p4_describe(changelist)
 destination = config.commit_notice_destination(changeroot, changelist)
 if destination is None:
    # Don't forward the notice anywhere
    sys.exit(0)
 message = """**{0}** committed revision @{1} to `{2}`.
 > {3}
 """.format(metadata["user"], metadata["change"], changeroot, metadata["desc"])
 message_data = {
    "type": "stream",
    "to": destination["stream"],
    "subject": destination["subject"],
    "content": message,
 }
 client.send_message(message_data)
--- a/api/integrations/perforce/zulip_perforce_config.py
+++ b/api/integrations/perforce/zulip_perforce_config.py
@@ -0,0 +1,63 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Change these values to configure authentication for the plugin
 ZULIP_USER = "p4-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # commit_notice_destination() lets you customize where commit notices
 # are sent to with the full power of a Python function.
 #
 # It takes the following arguments:
 # * path   = the path to the Perforce depot on the server
 # * changelist = the changelist id
 #
 # Returns a dictionary encoding the stream and topic to send the
 # notification to (or None to send no notification).
 #
 # The default code below will send every commit except for ones in the
 # "master-plan" and "secret" subdirectories of //depot/ to:
 # * stream "depot_subdirectory-commits"
 # * subject "change_root"
 def commit_notice_destination(path, changelist):
    dirs = path.split('/')
    if len(dirs) >= 4 and dirs[3] not in ("*", "..."):
        directory = dirs[3]
    else:
        # No subdirectory, so just use "depot"
        directory = dirs[2]
    if directory not in ["evil-master-plan", "my-super-secret-repository"]:
        return dict(stream  = "%s-commits" % (directory,),
                    subject = path)
    # Return None for cases where you don't want a notice sent
    return None
 ## If properly installed, the Zulip API should be in your import
 ## path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # This should not need to change unless you have a custom Zulip subdomain.
 ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/rss/rss-bot
+++ b/api/integrations/rss/rss-bot
@@ -0,0 +1,218 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # RSS integration for Zulip
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import calendar
 import errno
 import hashlib
 from HTMLParser import HTMLParser
 import logging
 import optparse
 import os
 import sys
 import time
 import urlparse
 import feedparser
 import zulip
 VERSION = "0.9"
 RSS_DATA_DIR = os.path.expanduser(os.path.join('~', '.cache', 'zulip-rss'))
 OLDNESS_THRESHOLD = 30 # days
 usage = """Usage: Send summaries of RSS entries for your favorite feeds to Zulip.
 This bot requires the feedparser module.
 To use this script:
 1. Create an RSS feed file containing 1 feed URL per line (default feed
   file location: ~/.cache/zulip-rss/rss-feeds)
 2. Subscribe to the stream that will receive RSS updates (default stream: rss)
 3. create a ~/.zuliprc as described on https://zulip.com/api#api_keys
 4. Test the script by running it manually, like this:
 /usr/local/share/zulip/integrations/rss/rss-bot
 You can customize the location on the feed file and recipient stream, e.g.:
 /usr/local/share/zulip/integrations/rss/rss-bot --feed-file=/path/to/my-feeds --stream=my-rss-stream
 4. Configure a crontab entry for this script. A sample crontab entry for
 processing feeds stored in the default location and sending to the default
 stream every 5 minutes is:
 */5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
 parser = optparse.OptionParser(usage)
 parser.add_option('--stream',
                  dest='stream',
                  help='The stream to which to send RSS messages.',
                  default="rss",
                  action='store')
 parser.add_option('--data-dir',
                  dest='data_dir',
                  help='The directory where feed metadata is stored',
                  default=os.path.join(RSS_DATA_DIR),
                  action='store')
 parser.add_option('--feed-file',
                  dest='feed_file',
                  help='The file containing a list of RSS feed URLs to follow, one URL per line',
                  default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
                  action='store')
 parser.add_option_group(zulip.generate_option_group(parser))
 (opts, args) = parser.parse_args()
 def mkdir_p(path):
    # Python doesn't have an analog to `mkdir -p` < Python 3.2.
    try:
        os.makedirs(path)
    except OSError, e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise
 try:
    mkdir_p(opts.data_dir)
 except OSError:
    # We can't write to the logfile, so just print and give up.
    print >>sys.stderr, "Unable to store RSS data at %s." % (opts.data_dir,)
    exit(1)
 log_file = os.path.join(opts.data_dir, "rss-bot.log")
 log_format = "%(asctime)s: %(message)s"
 logging.basicConfig(format=log_format)
 formatter = logging.Formatter(log_format)
 file_handler = logging.FileHandler(log_file)
 file_handler.setFormatter(formatter)
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 logger.addHandler(file_handler)
 def log_error_and_exit(error):
    logger.error(error)
    logger.error(usage)
    exit(1)
 class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, data):
        self.fed.append(data)
    def get_data(self):
        return ''.join(self.fed)
 def strip_tags(html):
    stripper = MLStripper()
    stripper.feed(html)
    return stripper.get_data()
 def compute_entry_hash(entry):
    entry_time = entry.get("published", entry.get("updated"))
    entry_id = entry.get("id", entry.get("link"))
    return hashlib.md5(entry_id + entry_time).hexdigest()
 def elide_subject(subject):
    MAX_TOPIC_LENGTH = 60
    if len(subject) > MAX_TOPIC_LENGTH:
        subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
    return subject
 def send_zulip(entry, feed_name):
    content = "**[%s](%s)**\n%s\n%s" % (entry.title,
                                  entry.link,
                                  strip_tags(entry.summary),
                                  entry.link)
    message = {"type": "stream",
               "sender": opts.email,
               "to": opts.stream,
               "subject": elide_subject(feed_name),
               "content": content,
               }
    return client.send_message(message)
 try:
    with open(opts.feed_file, "r") as f:
        feed_urls = [feed.strip() for feed in f.readlines()]
 except IOError:
    log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
 client = zulip.Client(email=opts.email, api_key=opts.api_key,
                       site=opts.site, client="ZulipRSS/" + VERSION)
 first_message = True
 for feed_url in feed_urls:
    feed_file = os.path.join(opts.data_dir, urlparse.urlparse(feed_url).netloc)
    try:
        with open(feed_file, "r") as f:
            old_feed_hashes = dict((line.strip(), True) for line in f.readlines())
    except IOError:
        old_feed_hashes = {}
    new_hashes = []
    data = feedparser.parse(feed_url)
    for entry in data.entries:
        entry_hash = compute_entry_hash(entry)
        # An entry has either been published or updated.
        entry_time  = entry.get("published_parsed", entry.get("updated_parsed"))
        if (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
            # As a safeguard against misbehaving feeds, don't try to process
            # entries older than some threshold.
            continue
        if entry_hash in old_feed_hashes:
            # We've already seen this. No need to process any older entries.
            break
        if (not old_feed_hashes) and (len(new_hashes) >= 3):
            # On a first run, pick up the 3 most recent entries. An RSS feed has
            # entries in reverse chronological order.
            break
        feed_name = data.feed.title or feed_url
        response = send_zulip(entry, feed_name)
        if response["result"] != "success":
            logger.error("Error processing %s" % (feed_url,))
            logger.error(response)
            if first_message:
                # This is probably some fundamental problem like the stream not
                # existing or something being misconfigured, so bail instead of
                # getting the same error for every RSS entry.
                log_error_and_exit("Failed to process first message")
        # Go ahead and move on -- perhaps this entry is corrupt.
        new_hashes.append(entry_hash)
        first_message = False
    with open(feed_file, "a") as f:
        for hash in new_hashes:
            f.write(hash + "\n")
    logger.info("Sent zulips for %d %s entries" % (len(new_hashes), feed_url))
--- a/api/integrations/svn/post-commit
+++ b/api/integrations/svn/post-commit
@@ -0,0 +1,71 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Zulip notification post-commit hook.
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
 # The "post-commit" script is run after a transaction is completed and a new
 # revision is created.  It is passed arguments on the command line in this
 # form:
 #  <path> <revision>
 # For example:
 # /srv/svn/carols 1843
 import os
 import sys
 import os.path
 import pysvn
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_svn_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipSVN/" + VERSION)
 svn = pysvn.Client()
 path, rev = sys.argv[1:]
 # since its a local path, prepend "file://"
 path = "file://" + path
 entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0]
 message = """**{0}** committed revision r{1} to `{2}`.
 > {3}
 """.format(entry['author'], rev, path.split('/')[-1], entry['revprops']['svn:log'])
 destination = config.commit_notice_destination(path, rev)
 message_data = {
    "type": "stream",
    "to": destination["stream"],
    "subject": destination["subject"],
    "content": message,
 }
 client.send_message(message_data)
--- a/api/integrations/svn/zulip_svn_config.py
+++ b/api/integrations/svn/zulip_svn_config.py
@@ -0,0 +1,57 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Change these values to configure authentication for the plugin
 ZULIP_USER = "svn-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # commit_notice_destination() lets you customize where commit notices
 # are sent to with the full power of a Python function.
 #
 # It takes the following arguments:
 # * path   = the path to the svn repository on the server
 # * commit = the commit id
 #
 # Returns a dictionary encoding the stream and subject to send the
 # notification to (or None to send no notification).
 #
 # The default code below will send every commit except for the "evil-master-plan"
 # and "my-super-secret-repository" repos to
 # * stream "commits"
 # * topic "branch_name"
 def commit_notice_destination(path, commit):
    repo = path.split('/')[-1]
    if repo not in ["evil-master-plan", "my-super-secret-repository"]:
        return dict(stream  = "commits",
                    subject = u"%s" % (repo,))
    # Return None for cases where you don't want a notice sent
    return None
 ## If properly installed, the Zulip API should be in your import
 ## path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/trac/zulip_trac.py
+++ b/api/integrations/trac/zulip_trac.py
@@ -0,0 +1,126 @@
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # Zulip trac plugin -- sends zulips when tickets change.
 #
 # Install by copying this file and zulip_trac_config.py to the trac
 # plugins/ subdirectory, customizing the constants in
 # zulip_trac_config.py, and then adding "zulip_trac" to the
 # components section of the conf/trac.ini file, like so:
 #
 # [components]
 # zulip_trac = enabled
 #
 # You may then need to restart trac (or restart Apache) for the bot
 # (or changes to the bot) to actually be loaded by trac.
 from trac.core import Component, implements
 from trac.ticket import ITicketChangeListener
 import sys
 import os.path
 sys.path.insert(0, os.path.dirname(__file__))
 import zulip_trac_config as config
 VERSION = "0.9"
 if config.ZULIP_API_PATH is not None:
    sys.path.append(config.ZULIP_API_PATH)
 import zulip
 client = zulip.Client(
    email=config.ZULIP_USER,
    site=config.ZULIP_SITE,
    api_key=config.ZULIP_API_KEY,
    client="ZulipTrac/" + VERSION)
 def markdown_ticket_url(ticket, heading="ticket"):
    return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
 def markdown_block(desc):
    return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
 def truncate(string, length):
    if len(string) <= length:
        return string
    return string[:length - 3] + "..."
 def trac_subject(ticket):
    return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
 def send_update(ticket, content):
    client.send_message({
            "type": "stream",
            "to": config.STREAM_FOR_NOTIFICATIONS,
            "content": content,
            "subject": trac_subject(ticket)
            })
 class ZulipPlugin(Component):
    implements(ITicketChangeListener)
    def ticket_created(self, ticket):
        """Called when a ticket is created."""
        content = "%s created %s in component **%s**, priority **%s**:\n" % \
            (ticket.values.get("reporter"), markdown_ticket_url(ticket),
             ticket.values.get("component"), ticket.values.get("priority"))
        # Include the full subject if it will be truncated
        if len(ticket.values.get("summary")) > 60:
            content += "**%s**\n" % (ticket.values.get("summary"),)
        if ticket.values.get("description") != "":
            content += "%s" % (markdown_block(ticket.values.get("description")),)
        send_update(ticket, content)
    def ticket_changed(self, ticket, comment, author, old_values):
        """Called when a ticket is modified.
        `old_values` is a dictionary containing the previous values of the
        fields that have changed.
        """
        if not (set(old_values.keys()).intersection(set(config.TRAC_NOTIFY_FIELDS)) or
                (comment and "comment" in set(config.TRAC_NOTIFY_FIELDS))):
            return
        content = "%s updated %s" % (author, markdown_ticket_url(ticket))
        if comment:
            content += ' with comment: %s\n\n' % (markdown_block(comment),)
        else:
            content += ":\n\n"
        field_changes = []
        for key in old_values.keys():
            if key == "description":
                content += '- Changed %s from %s to %s' % (key, markdown_block(old_values.get(key)),
                                                           markdown_block(ticket.values.get(key)))
            elif old_values.get(key) == "":
                field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
            elif ticket.values.get(key) == "":
                field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
            else:
                field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
                                                               ticket.values.get(key)))
        content += ", ".join(field_changes)
        send_update(ticket, content)
    def ticket_deleted(self, ticket):
        """Called when a ticket is deleted."""
        content = "%s was deleted." % markdown_ticket_url(ticket, heading="Ticket")
        send_update(ticket, content)
--- a/api/integrations/trac/zulip_trac_config.py
+++ b/api/integrations/trac/zulip_trac_config.py
@@ -0,0 +1,51 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 # See zulip_trac.py for installation and configuration instructions
 # Change these constants to configure the plugin:
 ZULIP_USER = "trac-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 STREAM_FOR_NOTIFICATIONS = "trac"
 TRAC_BASE_TICKET_URL = "https://trac.example.com/ticket"
 # Most people find that having every change in Trac result in a
 # notification is too noisy -- in particular, when someone goes
 # through recategorizing a bunch of tickets, that can often be noisy
 # and annoying.  We solve this issue by only sending a notification
 # for changes to the fields listed below.
 #
 # Total list of possible fields is:
 # (priority, milestone, cc, owner, keywords, component, severity,
 #  type, versions, description, resolution, summary, comment)
 #
 # The following is the list of fields which can be changed without
 # triggering a Zulip notification; change these to match your team's
 # workflow.
 TRAC_NOTIFY_FIELDS = ["description", "summary", "resolution", "comment", "owner"]
 ## If properly installed, the Zulip API should be in your import
 ## path, but if not, set a custom path below
 ZULIP_API_PATH = None
 # If you're using Zulip Enterprise, set this to your Zulip Enterprise server
 ZULIP_SITE = "https://api.zulip.com"
--- a/api/integrations/twitter/twitter-bot
+++ b/api/integrations/twitter/twitter-bot
@@ -0,0 +1,162 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Twitter integration for Zulip
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import os
 import sys
 import optparse
 import ConfigParser
 import zulip
 VERSION = "0.9"
 CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
 def write_config(config, since_id, user):
    config.set('twitter', 'since_id', since_id)
    config.set('twitter', 'user_id', user)
    with open(CONFIGFILE, 'wb') as configfile:
        config.write(configfile)
 parser = optparse.OptionParser(r"""
 %prog --user foo@example.com --api-key 0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --twitter-id twitter_handle
    Slurp tweets on your timeline into a specific zulip stream.
    Run this on your personal machine.  Your API key and twitter id are revealed to local
    users through the command line or config file.
    This bot uses OAuth to authenticate with twitter. Please create a ~/.zulip_twitterrc with
    the following contents:
    [twitter]
    consumer_key =
    consumer_secret =
    access_token_key =
    access_token_secret =
    In order to obtain a consumer key & secret, you must register a new application under your twitter account:
    1. Go to http://dev.twitter.com
    2. Log in
    3. In the menu under your username, click My Applications
    4. Create a new application
    Make sure to go the application you created and click "create my access token" as well. Fill in the values displayed.
    Depends on: twitter-python
 """)
 parser.add_option('--twitter-id',
    help='Twitter username to poll for new tweets from"',
    metavar='URL')
 parser.add_option('--stream',
    help='Default zulip stream to write tweets to')
 parser.add_option('--limit-tweets',
    default=15,
    type='int',
    help='Maximum number of tweets to push at once')
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 if not options.twitter_id:
    parser.error('You must specify --twitter-id')
 try:
    config = ConfigParser.ConfigParser()
    config.read(CONFIGFILE)
    consumer_key = config.get('twitter', 'consumer_key')
    consumer_secret = config.get('twitter', 'consumer_secret')
    access_token_key = config.get('twitter', 'access_token_key')
    access_token_secret = config.get('twitter', 'access_token_secret')
 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
   parser.error("Please provide a ~/.zulip_twitterrc")
 if not consumer_key or not consumer_secret or not access_token_key or not access_token_secret:
   parser.error("Please provide a ~/.zulip_twitterrc")
 try:
    import twitter
 except ImportError:
    parser.error("Please install twitter-python")
 api = twitter.Api(consumer_key=consumer_key,
                  consumer_secret=consumer_secret,
                  access_token_key=access_token_key,
                  access_token_secret=access_token_secret)
 user = api.VerifyCredentials()
 if not user.GetId():
    print "Unable to log in to twitter with supplied credentials. Please double-check and try again"
    sys.exit()
 try:
    since_id = config.getint('twitter', 'since_id')
 except ConfigParser.NoOptionError:
    since_id = -1
 try:
    user_id = config.get('twitter', 'user_id')
 except ConfigParser.NoOptionError:
    user_id = options.twitter_id
 client = zulip.Client(
    email=options.email,
    api_key=options.api_key,
    site=options.site,
    client="ZulipTwitter/" + VERSION,
    verbose=True)
 if since_id < 0 or options.twitter_id != user_id:
    # No since id yet, fetch the latest and then start monitoring from next time
    # Or, a different user id is being asked for, so start from scratch
    # Either way, fetch last 5 tweets to start off
    statuses = api.GetFriendsTimeline(user=options.twitter_id, count=5)
 else:
    # We have a saved last id, so insert all newer tweets into the zulip stream
    statuses = api.GetFriendsTimeline(user=options.twitter_id, since_id=since_id)
 for status in statuses[::-1][:options.limit_tweets]:
    composed = "%s (%s)" % (status.GetUser().GetName(), status.GetUser().GetScreenName())
    message = {
      "type": "stream",
      "to": [options.stream],
      "subject": composed,
      "content": status.GetText(),
    }
    ret = client.send_message(message)
    if ret['result'] == 'error':
        # If sending failed (e.g. no such stream), abort and retry next time
        print "Error sending message to zulip: %s" % ret['msg']
        break
    else:
        since_id = status.GetId()
 write_config(config, since_id, user_id)
--- a/api/integrations/twitter/twitter-search-bot
+++ b/api/integrations/twitter/twitter-search-bot
@@ -0,0 +1,190 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Twitter search integration for Zulip
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import os
 import sys
 import optparse
 import ConfigParser
 import zulip
 VERSION = "0.9"
 CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
 def write_config(config, since_id):
    if 'search' not in config.sections():
        config.add_section('search')
    config.set('search', 'since_id', since_id)
    with open(CONFIGFILE, 'wb') as configfile:
        config.write(configfile)
 parser = optparse.OptionParser(r"""
 %prog --user foo@zulip.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --search="@nprnews,quantum physics"
 Send Twitter search results to a Zulip stream.
 Depends on: twitter-python version 1.0 or later
 To use this script:
 1. Set up Twitter authentication, as described below
 2. Subscribe to the stream that will receive Twitter updates (default stream: twitter)
 3. Test the script by running it manually, like this:
 /usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
 4. Configure a crontab entry for this script. A sample crontab entry
 that will process tweets every 5 minutes is:
 */5 * * * * /usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
 == Setting up Twitter authentications ==
 Run this on a personal or trusted machine, because your API key is
 visible to local users through the command line or config file.
 This bot uses OAuth to authenticate with Twitter. Please create a
 ~/.zulip_twitterrc with the following contents:
 [twitter]
 consumer_key =
 consumer_secret =
 access_token_key =
 access_token_secret =
 In order to obtain a consumer key & secret, you must register a
 new application under your Twitter account:
 1. Go to http://dev.twitter.com
 2. Log in
 3. In the menu under your username, click My Applications
 4. Create a new application
 Make sure to go the application you created and click "create my
 access token" as well. Fill in the values displayed.
 """)
 parser.add_option('--search',
                  dest='search_terms',
                  help='Terms to search on',
                  action='store')
 parser.add_option('--stream',
                  dest='stream',
                  help='The stream to which to send tweets',
                  default="twitter",
                  action='store')
 parser.add_option('--limit-tweets',
                  default=15,
                  type='int',
                  help='Maximum number of tweets to send at once')
 parser.add_option_group(zulip.generate_option_group(parser))
 (opts, args) = parser.parse_args()
 if not opts.search_terms:
    parser.error('You must specify a search term.')
 try:
    config = ConfigParser.ConfigParser()
    config.read(CONFIGFILE)
    consumer_key = config.get('twitter', 'consumer_key')
    consumer_secret = config.get('twitter', 'consumer_secret')
    access_token_key = config.get('twitter', 'access_token_key')
    access_token_secret = config.get('twitter', 'access_token_secret')
 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
   parser.error("Please provide a ~/.zulip_twitterrc")
 if not (consumer_key and consumer_secret and access_token_key and access_token_secret):
   parser.error("Please provide a ~/.zulip_twitterrc")
 try:
    since_id = config.getint('search', 'since_id')
 except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
    since_id = 0
 try:
    import twitter
 except ImportError:
    parser.error("Please install twitter-python")
 api = twitter.Api(consumer_key=consumer_key,
                  consumer_secret=consumer_secret,
                  access_token_key=access_token_key,
                  access_token_secret=access_token_secret)
 user = api.VerifyCredentials()
 if not user.GetId():
    print "Unable to log in to Twitter with supplied credentials.\
 Please double-check and try again."
    sys.exit()
 client = zulip.Client(
    email=opts.email,
    api_key=opts.api_key,
    site=opts.site,
    client="ZulipTwitterSearch/" + VERSION,
    verbose=True)
 search_query = " OR ".join(opts.search_terms.split(","))
 statuses = api.GetSearch(search_query, since_id=since_id)
 for status in statuses[::-1][:opts.limit_tweets]:
    # https://twitter.com/eatevilpenguins/status/309995853408530432
    composed = "%s (%s)" % (status.GetUser().GetName(),
                            status.GetUser().GetScreenName())
    url = "https://twitter.com/%s/status/%s" % (status.GetUser().GetScreenName(),
                                                status.GetId())
    content = status.GetText()
    search_term_used = None
    for term in opts.search_terms.split(","):
        if term.lower() in content.lower():
            search_term_used = term
            break
    # For some reason (perhaps encodings or message tranformations we
    # didn't anticipate), we don't know what term was used, so use a
    # default.
    if not search_term_used:
        search_term_used = "mentions"
    message = {
      "type": "stream",
      "to": [opts.stream],
      "subject": search_term_used,
      "content": url,
    }
    ret = client.send_message(message)
    if ret['result'] == 'error':
        # If sending failed (e.g. no such stream), abort and retry next time
        print "Error sending message to zulip: %s" % ret['msg']
        break
    else:
        since_id = status.GetId()
 write_config(config, since_id)
--- a/api/setup.cfg
+++ b/api/setup.cfg
@@ -0,0 +1,3 @@
 # This way our scripts are installed in the data directory
 [aliases]
 install = install_data install
--- a/api/setup.py
+++ b/api/setup.py
@@ -0,0 +1,73 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
 import sys
 import itertools
 def version():
    version_py = os.path.join(os.path.dirname(__file__), "zulip", "__init__.py")
    with open(version_py) as in_handle:
        version_line = itertools.dropwhile(lambda x: not x.startswith("__version__"),
                                           in_handle).next()
    version = version_line.split('=')[-1].strip().replace('"', '')
    return version
 def recur_expand(target_root, dir):
    for root, _, files in os.walk(dir):
        paths = [os.path.join(root, f) for f in files]
        if len(paths):
            yield os.path.join(target_root, root), paths
 # We should be installable with either setuptools or distutils.
 package_info = dict(
    name='zulip',
    version=version(),
    description='Bindings for the Zulip message API',
    author='Zulip, Inc.',
    author_email='support@zulip.com',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Topic :: Communications :: Chat',
    ],
    url='https://www.zulip.com/dist/api/',
    packages=['zulip'],
    data_files=[('share/zulip/examples', ["examples/zuliprc", "examples/send-message", "examples/subscribe",
                                           "examples/get-public-streams", "examples/unsubscribe",
                                           "examples/list-members", "examples/list-subscriptions",
                                           "examples/print-messages"])] + \
        list(recur_expand('share/zulip', 'integrations/')),
    scripts=["bin/zulip-send"],
 )
 setuptools_info = dict(
    install_requires=['requests>=0.12.1',
                      'simplejson',
    ],
 )
 try:
    from setuptools import setup
    package_info.update(setuptools_info)
 except ImportError:
    from distutils.core import setup
    from distutils.version import LooseVersion
    # Manual dependency check
    try:
        import simplejson
    except ImportError:
        print >>sys.stderr, "simplejson is not installed"
        sys.exit(1)
    try:
        import requests
        assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
    except (ImportError, AssertionError):
        print >>sys.stderr, "requests >=0.12.1 is not installed"
        sys.exit(1)
 setup(**package_info)
--- a/api/zulip/init.py
+++ b/api/zulip/init.py
@@ -0,0 +1,363 @@
 # -*- coding: utf-8 -*-
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import simplejson
 import requests
 import time
 import traceback
 import urlparse
 import sys
 import os
 import optparse
 import platform
 from distutils.version import LooseVersion
 from ConfigParser import SafeConfigParser
 import logging
 __version__ = "0.2.4"
 # Check that we have a recent enough version
 # Older versions don't provide the 'json' attribute on responses.
 assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
 # In newer versions, the 'json' attribute is a function, not a property
 requests_json_is_function = callable(requests.Response.json)
 API_VERSTRING = "v1/"
 def _default_client():
    return "ZulipPython/" + __version__
 def generate_option_group(parser):
    group = optparse.OptionGroup(parser, 'API configuration')
    group.add_option('--site',
                     help="Zulip Enterprise server URI (if using Zulip Enterprise)",
                     default=None)
    group.add_option('--api-key',
                     action='store')
    group.add_option('--user',
                     dest='email',
                     help='Email address of the calling bot or user.')
    group.add_option('--config-file',
                     action='store',
                     help='Location of an ini file containing the\nabove information. (default ~/.zuliprc)')
    group.add_option('-v', '--verbose',
                     action='store_true',
                     help='Provide detailed output.')
    group.add_option('--client',
                     action='store',
                     default=None,
                     help=optparse.SUPPRESS_HELP)
    return group
 def init_from_options(options, client=None):
    if options.client is not None:
        client = options.client
    elif client is None:
        client = _default_client()
    return Client(email=options.email, api_key=options.api_key,
                  config_file=options.config_file, verbose=options.verbose,
                  site=options.site, client=client)
 class Client(object):
    def __init__(self, email=None, api_key=None, config_file=None,
                 verbose=False, retry_on_errors=True,
                 site=None, client=None):
        if client is None:
            client = _default_client()
        if None in (api_key, email):
            if config_file is None:
                config_file = os.path.join(os.environ["HOME"], ".zuliprc")
                if (not os.path.exists(config_file) and
                    os.path.exists(os.path.join(os.environ["HOME"], ".humbugrc"))):
                    raise RuntimeError("The Zulip API configuration file is now ~/.zuliprc; please run:\n\n  mv ~/.humbugrc ~/.zuliprc\n")
            if not os.path.exists(config_file):
                raise RuntimeError("api_key or email not specified and %s does not exist"
                                   % (config_file,))
            config = SafeConfigParser()
            with file(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            self.base_url = "https://api.zulip.com"
        if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client
    def get_user_agent(self):
        vendor = platform.system()
        vendor_version = platform.release()
        if vendor == "Linux":
            vendor, vendor_version, dummy = platform.linux_distribution()
        elif vendor == "Windows":
            vendor_version = platform.win32_ver()[1]
        elif vendor == "Darwin":
            vendor_version = platform.mac_ver()[0]
        return "{client_name} ({vendor}; {vendor_version})".format(
                client_name=self.client_name,
                vendor=vendor,
                vendor_version=vendor_version,
                )
    def do_api_query(self, orig_request, url, method="POST", longpolling = False):
        request = {}
        for (key, val) in orig_request.iteritems():
            if not (isinstance(val, str) or isinstance(val, unicode)):
                request[key] = simplejson.dumps(val)
            else:
                request[key] = val
        query_state = {
            'had_error_retry': False,
            'request': request,
            'failures': 0,
        }
        def error_retry(error_string):
            if not self.retry_on_errors or query_state["failures"] >= 10:
                return False
            if self.verbose:
                if not query_state["had_error_retry"]:
                    sys.stdout.write("zulip API(%s): connection error%s -- retrying." % \
                            (url.split(API_VERSTRING, 2)[0], error_string,))
                    query_state["had_error_retry"] = True
                else:
                    sys.stdout.write(".")
                sys.stdout.flush()
            query_state["request"]["dont_block"] = simplejson.dumps(True)
            time.sleep(1)
            query_state["failures"] += 1
            return True
        def end_error_retry(succeeded):
            if query_state["had_error_retry"] and self.verbose:
                if succeeded:
                    print "Success!"
                else:
                    print "Failed!"
        while True:
            try:
                if method == "GET":
                    kwarg = "params"
                else:
                    kwarg = "data"
                kwargs = {kwarg: query_state["request"]}
                res = requests.request(
                        method,
                        urlparse.urljoin(self.base_url, url),
                        auth=requests.auth.HTTPBasicAuth(self.email,
                                                         self.api_key),
                        verify=True, timeout=90,
                        headers={"User-agent": self.get_user_agent()},
                        **kwargs)
                # On 50x errors, try again after a short sleep
                if str(res.status_code).startswith('5'):
                    if error_retry(" (server %s)" % (res.status_code,)):
                        continue
                    # Otherwise fall through and process the python-requests error normally
            except (requests.exceptions.Timeout, requests.exceptions.SSLError) as e:
                # Timeouts are either a Timeout or an SSLError; we
                # want the later exception handlers to deal with any
                # non-timeout other SSLErrors
                if (isinstance(e, requests.exceptions.SSLError) and
                    str(e) != "The read operation timed out"):
                    raise
                if longpolling:
                    # When longpolling, we expect the timeout to fire,
                    # and the correct response is to just retry
                    continue
                else:
                    end_error_retry(False)
                    return {'msg': "Connection error:\n%s" % traceback.format_exc(),
                            "result": "connection-error"}
            except requests.exceptions.ConnectionError:
                if error_retry(""):
                    continue
                end_error_retry(False)
                return {'msg': "Connection error:\n%s" % traceback.format_exc(),
                        "result": "connection-error"}
            except Exception:
                # We'll split this out into more cases as we encounter new bugs.
                return {'msg': "Unexpected error:\n%s" % traceback.format_exc(),
                        "result": "unexpected-error"}
            try:
                if requests_json_is_function:
                    json_result = res.json()
                else:
                    json_result = res.json
            except Exception:
                json_result = None
            if json_result is not None:
                end_error_retry(True)
                return json_result
            end_error_retry(False)
            return {'msg': "Unexpected error from the server", "result": "http-error",
                    "status_code": res.status_code}
    @classmethod
    def _register(cls, name, url=None, make_request=(lambda request={}: request),
            method="POST", **query_kwargs):
        if url is None:
            url = name
        def call(self, *args, **kwargs):
            request = make_request(*args, **kwargs)
            return self.do_api_query(request, API_VERSTRING + url, method=method, **query_kwargs)
        call.func_name = name
        setattr(cls, name, call)
    def call_on_each_event(self, callback, event_types=None, narrow=[]):
        def do_register():
            while True:
                if event_types is None:
                    res = self.register()
                else:
                    res = self.register(event_types=event_types, narrow=narrow)
                if 'error' in res.get('result'):
                    if self.verbose:
                        print "Server returned error:\n%s" % res['msg']
                    time.sleep(1)
                else:
                    return (res['queue_id'], res['last_event_id'])
        queue_id = None
        while True:
            if queue_id is None:
                (queue_id, last_event_id) = do_register()
            res = self.get_events(queue_id=queue_id, last_event_id=last_event_id)
            if 'error' in res.get('result'):
                if res["result"] == "http-error":
                    if self.verbose:
                        print "HTTP error fetching events -- probably a server restart"
                elif res["result"] == "connection-error":
                    if self.verbose:
                        print "Connection error fetching events -- probably server is temporarily down?"
                else:
                    if self.verbose:
                        print "Server returned error:\n%s" % res["msg"]
                    if res["msg"].startswith("Bad event queue id:"):
                        # Our event queue went away, probably because
                        # we were asleep or the server restarted
                        # abnormally.  We may have missed some
                        # events while the network was down or
                        # something, but there's not really anything
                        # we can do about it other than resuming
                        # getting new ones.
                        #
                        # Reset queue_id to register a new event queue.
                        queue_id = None
                # TODO: Make this back off once it's more reliable
                time.sleep(1)
                continue
            for event in res['events']:
                last_event_id = max(last_event_id, int(event['id']))
                callback(event)
    def call_on_each_message(self, callback):
        def event_callback(event):
            if event['type'] == 'message':
                callback(event['message'])
        self.call_on_each_event(event_callback, ['message'])
 def _mk_subs(streams, **kwargs):
    result = kwargs
    result['subscriptions'] = streams
    return result
 def _mk_rm_subs(streams):
    return {'delete': streams}
 def _mk_deregister(queue_id):
    return {'queue_id': queue_id}
 def _mk_events(event_types=None, narrow=[]):
    if event_types is None:
        return dict()
    return dict(event_types=event_types, narrow=narrow)
 def _kwargs_to_dict(**kwargs):
    return kwargs
 class ZulipStream(object):
    """
    A Zulip stream-like object
    """
    def __init__(self, type, to, subject, **kwargs):
        self.client = Client(**kwargs)
        self.type = type
        self.to = to
        self.subject = subject
    def write(self, content):
        message = {"type": self.type,
                   "to": self.to,
                   "subject": self.subject,
                   "content": content}
        self.client.send_message(message)
    def flush(self):
        pass
 Client._register('send_message', url='messages', make_request=(lambda request: request))
 Client._register('update_message', method='PATCH', url='messages', make_request=(lambda request: request))
 Client._register('get_messages', method='GET', url='messages/latest', longpolling=True)
 Client._register('get_events', url='events', method='GET', longpolling=True, make_request=(lambda **kwargs: kwargs))
 Client._register('register', make_request=_mk_events)
 Client._register('deregister', url="events", method="DELETE", make_request=_mk_deregister)
 Client._register('get_profile', method='GET', url='users/me')
 Client._register('get_streams', method='GET', url='streams', make_request=_kwargs_to_dict)
 Client._register('get_members', method='GET', url='users')
 Client._register('list_subscriptions', method='GET', url='users/me/subscriptions')
 Client._register('add_subscriptions', url='users/me/subscriptions', make_request=_mk_subs)
 Client._register('remove_subscriptions', method='PATCH', url='users/me/subscriptions', make_request=_mk_rm_subs)
 Client._register('render_message', method='GET', url='messages/render')
 Client._register('create_user', method='POST', url='users')
--- a/static/assets/README
+++ b/static/assets/README
--- a/static/assets/default-default-avatar.svg
+++ b/static/assets/default-default-avatar.svg
--- a/static/assets/favicon/.gitignore
+++ b/static/assets/favicon/.gitignore
--- a/Show More
+++ b/Show More
		`@@ -1,2 +0,0 @@`
			`static/js/blueslip.js`
			`static/webpack-bundles`