Release Zulip Server 1.7.1.

This completes the last commit's work to fix CVE-2017-0910, applying
to any invite links already created before the fix was deployed.  With
this change, all new-user registrations must match an explicit realm
in the PreregistrationUser row, except when creating a new realm.

[greg: rewrote commit message]

.browserslistrc

@@ -1,6 +0,0 @@
-> 0.2%
-> 0.2% in US
-last 2 versions
-Firefox ESR
-not dead
-Chrome 26  # similar to PhantomJS

.circleci/config.yml

@@ -1,383 +0,0 @@
-# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
-#   high-level documentation on our CircleCI setup.
-# See CircleCI upstream's docs on this config format:
-#   https://circleci.com/docs/2.0/language-python/
-#
-version: 2.0
-aliases:
-  - &create_cache_directories
-    run:
-      name: create cache directories
-      command: |
-        dirs=(/srv/zulip-{npm,venv,emoji}-cache)
-        sudo mkdir -p "${dirs[@]}"
-        sudo chown -R circleci "${dirs[@]}"
-
-  - &restore_cache_package_json
-    restore_cache:
-      keys:
-        - v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-
-  - &restore_cache_requirements
-    restore_cache:
-      keys:
-        - v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor-dev.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-  - &restore_emoji_cache
-    restore_cache:
-      keys:
-        - v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "tools/setup/emoji/emoji_map.json" }}-{{ checksum "tools/setup/emoji/build_emoji" }}-{{checksum "tools/setup/emoji/emoji_setup_utils.py" }}-{{ checksum "tools/setup/emoji/emoji_names.py" }}-{{ checksum "package.json" }}
-
-  - &install_dependencies
-    run:
-      name: install dependencies
-      command: |
-        sudo apt-get update
-        # Install moreutils so we can use `ts` and `mispipe` in the following.
-        sudo apt-get install -y moreutils
-
-        # CircleCI sets the following in Git config at clone time:
-        #   url.ssh://git@github.com.insteadOf https://github.com
-        # This breaks the Git clones in the NVM `install.sh` we run
-        # in `install-node`.
-        # TODO: figure out why that breaks, and whether we want it.
-        #   (Is it an optimization?)
-        rm -f /home/circleci/.gitconfig
-
-        # This is the main setup job for the test suite
-        mispipe "tools/ci/setup-backend --skip-dev-db-build" ts
-
-        # Cleaning caches is mostly unnecessary in Circle, because
-        # most builds don't get to write to the cache.
-        # mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0 2>&1" ts
-
-  - &save_cache_package_json
-    save_cache:
-      paths:
-        - /srv/zulip-npm-cache
-      key: v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
-
-  - &save_cache_requirements
-    save_cache:
-      paths:
-        - /srv/zulip-venv-cache
-      key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor-dev.txt" }}-{{ checksum "requirements/dev.txt" }}
-
-  - &save_emoji_cache
-    save_cache:
-      paths:
-        - /srv/zulip-emoji-cache
-      key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "tools/setup/emoji/emoji_map.json" }}-{{ checksum "tools/setup/emoji/build_emoji" }}-{{checksum "tools/setup/emoji/emoji_setup_utils.py" }}-{{ checksum "tools/setup/emoji/emoji_names.py" }}-{{ checksum "package.json" }}
-
-  - &do_bionic_hack
-    run:
-      name: do Bionic hack
-      command: |
-        # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
-        # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
-        sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
-
-  - &run_backend_tests
-    run:
-      name: run backend tests
-      command: |
-        . /srv/zulip-py3-venv/bin/activate
-        mispipe "./tools/ci/backend 2>&1" ts
-
-  - &run_frontend_tests
-    run:
-      name: run frontend tests
-      command: |
-        . /srv/zulip-py3-venv/bin/activate
-        mispipe "./tools/ci/frontend 2>&1" ts
-
-  - &upload_coverage_report
-    run:
-      name: upload coverage report
-      command: |
-        # codecov requires `.coverage` file to be stored in pwd for
-        # uploading coverage results.
-        mv /home/circleci/zulip/var/.coverage /home/circleci/zulip/.coverage
-
-        . /srv/zulip-py3-venv/bin/activate
-        # TODO: Check that the next release of codecov doesn't
-        # throw find error.
-        # codecov==2.0.16 introduced a bug which uses "find"
-        # for locating files which is buggy on some platforms.
-        # It was fixed via https://github.com/codecov/codecov-python/pull/217
-        # and should get automatically fixed here once it's released.
-        # We cannot pin the version here because we need the latest version for uploading files.
-        # see https://community.codecov.io/t/http-400-while-uploading-to-s3-with-python-codecov-from-travis/1428/7
-        pip install codecov && codecov \
-          || echo "Error in uploading coverage reports to codecov.io."
-
-  - &build_production
-    run:
-      name: build production
-      command: |
-        sudo apt-get update
-        # Install moreutils so we can use `ts` and `mispipe` in the following.
-        sudo apt-get install -y moreutils
-
-        mispipe "./tools/ci/production-build 2>&1" ts
-
-  - &production_extract_tarball
-    run:
-      name: production extract tarball
-      command: |
-        sudo apt-get update
-        # Install moreutils so we can use `ts` and `mispipe` in the following.
-        sudo apt-get install -y moreutils
-
-        mispipe "/tmp/production-extract-tarball 2>&1" ts
-
-  - &install_production
-    run:
-      name: install production
-      command: |
-        sudo service rabbitmq-server restart
-        sudo mispipe "/tmp/production-install 2>&1" ts
-
-  - &verify_production
-    run:
-      name: verify install
-      command: |
-        sudo mispipe "/tmp/production-verify 2>&1" ts
-
-  - &upgrade_postgresql
-    run:
-      name: upgrade postgresql
-      command: |
-        sudo mispipe "/tmp/production-upgrade-pg 2>&1" ts
-
-  - &check_xenial_provision_error
-    run:
-      name: check tools/provision error message on xenial
-      command: |
-        ! tools/provision > >(tee provision.out)
-        grep -Fqx 'CRITICAL:root:Unsupported platform: ubuntu 16.04' provision.out
-
-  - &check_xenial_upgrade_error
-    run:
-      name: check scripts/lib/upgrade-zulip-stage-2 error message on xenial
-      command: |
-        ! sudo scripts/lib/upgrade-zulip-stage-2 2> >(tee upgrade.err >&2)
-        grep -Fq 'upgrade-zulip-stage-2: Unsupported platform: ubuntu 16.04' upgrade.err
-
-  - &notify_failure_status
-    run:
-      name: On fail
-      when: on_fail
-      branches:
-        only: master
-      command: |
-        if [[ "$CIRCLE_REPOSITORY_URL" == "git@github.com:zulip/zulip.git" && "$ZULIP_BOT_KEY" != "" ]]; then
-          curl  -H "Content-Type: application/json" \
-          -X POST -i 'https://chat.zulip.org/api/v1/external/circleci?api_key='"$ZULIP_BOT_KEY"'&stream=automated%20testing&topic=master%20failing' \
-          -d '{"payload": { "branch": "'"$CIRCLE_BRANCH"'", "reponame": "'"$CIRCLE_PROJECT_REPONAME"'", "status": "failed", "build_url": "'"$CIRCLE_BUILD_URL"'", "username": "'"$CIRCLE_USERNAME"'"}}'
-        fi
-
-jobs:
-  "bionic-backend-frontend":
-    docker:
-      # This is built from tools/ci/images/bionic/Dockerfile .
-      # Bionic ships with Python 3.6.
-      - image: arpit551/circleci:bionic-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *create_cache_directories
-      - *do_bionic_hack
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *restore_emoji_cache
-      - *install_dependencies
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *save_emoji_cache
-      - *run_backend_tests
-
-      - run:
-          name: test locked requirements
-          command: |
-            . /srv/zulip-py3-venv/bin/activate
-            mispipe "./tools/test-locked-requirements 2>&1" ts
-
-      - *run_frontend_tests
-        # We only need to upload coverage reports on whichever platform
-        # runs the frontend tests.
-      - *upload_coverage_report
-
-      - store_artifacts:
-          path: ./var/casper/
-          destination: casper
-
-      - store_artifacts:
-          path: ./var/puppeteer/
-          destination: puppeteer
-
-      - store_artifacts:
-          path: ../../../tmp/zulip-test-event-log/
-          destination: test-reports
-
-      - store_test_results:
-          path: ./var/xunit-test-results/casper/
-      - *notify_failure_status
-
-  "focal-backend":
-    docker:
-      # This is built from tools/ci/images/focal/Dockerfile.
-      # Focal ships with Python 3.8.2.
-      - image: arpit551/circleci:focal-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *create_cache_directories
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *restore_emoji_cache
-      - *install_dependencies
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *save_emoji_cache
-      - *run_backend_tests
-      - run:
-          name: Check development database build
-          command: mispipe "tools/ci/setup-backend" ts
-      - *notify_failure_status
-
-  "xenial-legacy":
-    docker:
-      - image: arpit551/circleci:xenial-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *check_xenial_provision_error
-      - *check_xenial_upgrade_error
-      - *notify_failure_status
-
-  "bionic-production-build":
-    docker:
-      # This is built from tools/ci/images/bionic/Dockerfile .
-      # Bionic ships with Python 3.6.
-      - image: arpit551/circleci:bionic-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-
-      - *create_cache_directories
-      - *do_bionic_hack
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *restore_emoji_cache
-      - *build_production
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *save_emoji_cache
-
-      # Persist the built tarball to be used in downstream job
-      # for installation of production server.
-      # See https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
-      - persist_to_workspace:
-          # Must be an absolute path,
-          # or relative path from working_directory.
-          # This is a directory on the container which is
-          # taken to be the root directory of the workspace.
-          root: /tmp
-          # Must be relative path from root
-          paths:
-            - zulip-server-test.tar.gz
-            - success-http-headers.template.txt
-            - production-install
-            - production-verify
-            - production-upgrade-pg
-            - production
-            - production-extract-tarball
-      - *notify_failure_status
-
-  "bionic-production-install":
-    docker:
-      # This is built from tools/ci/images/bionic/Dockerfile .
-      # Bionic ships with Python 3.6.
-      - image: arpit551/circleci:bionic-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      # Contains the built tarball from bionic-production-build job
-      - attach_workspace:
-          # Must be absolute path or relative path from working_directory
-          at: /tmp
-
-      - *create_cache_directories
-      - *do_bionic_hack
-      - *production_extract_tarball
-      - *restore_cache_package_json
-      - *install_production
-      - *verify_production
-      - *upgrade_postgresql
-      - *verify_production
-      - *save_cache_package_json
-      - *notify_failure_status
-
-  "focal-production-install":
-    docker:
-      # This is built from tools/ci/images/focal/Dockerfile.
-      # Focal ships with Python 3.8.2.
-      - image: arpit551/circleci:focal-python-test
-
-    working_directory: ~/zulip
-
-    steps:
-      # Contains the built tarball from bionic-production-build job
-      - attach_workspace:
-          # Must be absolute path or relative path from working_directory
-          at: /tmp
-
-      - *create_cache_directories
-
-      - run:
-          name: do memcached hack
-          command: |
-            # Temporary hack till memcached upstream is updated in Focal.
-            # https://bugs.launchpad.net/ubuntu/+source/memcached/+bug/1878721
-            echo "export SASL_CONF_PATH=/etc/sasl2" | sudo tee - a /etc/default/memcached
-
-      - *production_extract_tarball
-      - *restore_cache_package_json
-      - *install_production
-      - *verify_production
-      - *save_cache_package_json
-      - *notify_failure_status
-
-workflows:
-  version: 2
-  "Ubuntu 16.04 Xenial (Python 3.5, legacy)":
-    jobs:
-      - "xenial-legacy"
-  "Ubuntu 18.04 Bionic (Python 3.6, backend+frontend)":
-    jobs:
-      - "bionic-backend-frontend"
-  "Ubuntu 20.04 Focal (Python 3.8, backend)":
-    jobs:
-      - "focal-backend"
-  "Production":
-    jobs:
-      - "bionic-production-build"
-      - "bionic-production-install":
-          requires:
-            - "bionic-production-build"
-      - "focal-production-install":
-          requires:
-            - "bionic-production-build"

.codecov.yml

@@ -5,8 +5,6 @@ coverage:
     project:
       default:
         target: auto
-        # Codecov has the tendency to report a lot of false negatives,
-        # so we basically suppress comments completely.
-        threshold: 50%
+        threshold: 0.03
         base: auto
     patch: off

.editorconfig

@@ -3,19 +3,17 @@ root = true
 [*]
 end_of_line = lf
 charset = utf-8
-indent_size = 4
-indent_style = space
 trim_trailing_whitespace = true
 insert_final_newline = true
 
-binary_next_line = true  # for shfmt
-switch_case_indent = true  # for shfmt
+[*.{sh,py,js,json,yml,xml,css,md,markdown,handlebars,html}]
+indent_style = space
+indent_size = 4
 
-[*.py]
-max_line_length = 110
-
-[*.{js,ts}]
-max_line_length = 100
-
-[*.{svg,rb,pp}]
+[*.{svg,rb,pp,pl}]
+indent_style = space
 indent_size = 2
+
+[*.{cfg}]
+indent_style = space
+indent_size = 8

.eslintignore

@@ -1,10 +1,2 @@
-# This is intended for generated files and vendored third-party files.
-# For our source code, instead of adding files here, consider using
-# specific eslint-disable comments in the files themselves.
-
-/docs/_build
-/static/generated
-/static/third
-/static/webpack-bundles
-/var
-/zulip-py3-venv
+static/js/blueslip.js
+static/webpack-bundles

.eslintrc.json

@@ -3,348 +3,325 @@
         "node": true,
         "es6": true
     },
-    "extends": [
-        "eslint:recommended",
-        "prettier"
-    ],
     "parserOptions": {
-        "ecmaVersion": 2019,
-        "warnOnUnsupportedTypeScriptVersion": false,
         "sourceType": "module"
     },
-    "plugins": [
-        "eslint-plugin-empty-returns"
-    ],
+    "globals": {
+        "$": false,
+        "_": false,
+        "jQuery": false,
+        "Spinner": false,
+        "Handlebars": false,
+        "XDate": false,
+        "zxcvbn": false,
+        "LazyLoad": false,
+        "Dropbox": false,
+        "SockJS": false,
+        "marked": false,
+        "moment": false,
+        "i18n": false,
+        "DynamicText": false,
+        "LightboxCanvas": false,
+        "bridge": false,
+        "page_params": false,
+        "attachments_ui": false,
+        "csrf_token": false,
+        "typeahead_helper": false,
+        "pygments_data": false,
+        "popovers": false,
+        "server_events": false,
+        "server_events_dispatch": false,
+        "ui": false,
+        "ui_report": false,
+        "ui_util": false,
+        "lightbox": false,
+        "stream_color": false,
+        "people": false,
+        "navigate": false,
+        "settings_account": false,
+        "settings_display": false,
+        "settings_notifications": false,
+        "settings_muting": false,
+        "settings_lab": false,
+        "settings_bots": false,
+        "settings_sections": false,
+        "settings_emoji": false,
+        "settings_org": false,
+        "settings_users": false,
+        "settings_streams": false,
+        "settings_filters": false,
+        "settings": false,
+        "resize": false,
+        "loading": false,
+        "typing": false,
+        "typing_events": false,
+        "typing_data": false,
+        "typing_status": false,
+        "sent_messages": false,
+        "compose": false,
+        "compose_actions": false,
+        "compose_state": false,
+        "compose_fade": false,
+        "overlays": false,
+        "stream_create": false,
+        "stream_edit": false,
+        "subs": false,
+        "stream_muting": false,
+        "stream_events": false,
+        "timerender": false,
+        "message_live_update": false,
+        "message_edit": false,
+        "reload": false,
+        "composebox_typeahead": false,
+        "search": false,
+        "topic_list": false,
+        "topic_generator": false,
+        "gear_menu": false,
+        "hashchange": false,
+        "hash_util": false,
+        "message_list": false,
+        "Filter": false,
+        "pointer": false,
+        "util": false,
+        "MessageListView": false,
+        "blueslip": false,
+        "rows": false,
+        "WinChan": false,
+        "muting_ui": false,
+        "Socket": false,
+        "channel": false,
+        "components": false,
+        "message_viewport": false,
+        "upload_widget": false,
+        "avatar": false,
+        "realm_icon": false,
+        "feature_flags": false,
+        "search_suggestion": false,
+        "notifications": false,
+        "message_flags": false,
+        "bot_data": false,
+        "top_left_corner": false,
+        "stream_sort": false,
+        "stream_list": false,
+        "stream_popover": false,
+        "narrow_state": false,
+        "narrow": false,
+        "admin_sections": false,
+        "admin": false,
+        "stream_data": false,
+        "topic_data": false,
+        "list_util": false,
+        "muting": false,
+        "Dict": false,
+        "unread": false,
+        "alert_words_ui": false,
+        "message_store": false,
+        "message_util": false,
+        "message_events": false,
+        "message_fetch": false,
+        "favicon": false,
+        "condense": false,
+        "list_render": false,
+        "floating_recipient_bar": false,
+        "tab_bar": false,
+        "emoji": false,
+        "presence": false,
+        "activity": false,
+        "invite": false,
+        "colorspace": false,
+        "reactions": false,
+        "tutorial": false,
+        "templates": false,
+        "alert_words": false,
+        "fenced_code": false,
+        "markdown": false,
+        "echo": false,
+        "localstorage": false,
+        "localStorage": false,
+        "current_msg_list": true,
+        "home_msg_list": false,
+        "pm_list": false,
+        "pm_conversations": false,
+        "recent_senders": false,
+        "unread_ui": false,
+        "unread_ops": false,
+        "user_events": false,
+        "Plotly": false,
+        "emoji_codes": false,
+        "drafts": false,
+        "katex": false,
+        "Clipboard": false,
+        "emoji_picker": false,
+        "hotspots": false,
+        "compose_ui": false,
+        "common": false,
+        "desktop_notifications_panel": false
+    },
     "rules": {
         "array-callback-return": "error",
-        "arrow-body-style": "error",
-        "block-scoped-var": "error",
-        "curly": "error",
-        "dot-notation": "error",
-        "empty-returns/main": "error",
-        "eqeqeq": "error",
-        "guard-for-in": "error",
+        "array-bracket-spacing": "error",
+        "arrow-spacing": [ "error", { "before": true, "after": true } ],
+        "block-scoped-var": 2,
+        "brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
+        "camelcase": 0,
+        "comma-dangle": [ "error",
+            {
+                "arrays": "always-multiline",
+                "objects": "always-multiline",
+                "imports": "always-multiline",
+                "exports": "always-multiline",
+                "functions": "never"
+            }
+        ],
+        "complexity": [ 0, 4 ],
+        "curly": 2,
+        "dot-notation": [ "error", { "allowKeywords": true } ],
+        "eol-last": [ "error", "always" ],
+        "eqeqeq": 2,
+        "func-style": [ "off", "expression" ],
+        "guard-for-in": 2,
+        "keyword-spacing": [ "error",
+            {
+                "before": true,
+                "after": true,
+                "overrides": {
+                    "return": { "after": true },
+                    "throw": { "after": true },
+                    "case": { "after": true }
+                }
+            }
+        ],
+        "max-depth": [ 0, 4 ],
+        "max-len": [ "error", 100, 2,
+            {
+                "ignoreUrls": true,
+                "ignoreComments": false,
+                "ignoreRegExpLiterals": true,
+                "ignoreStrings": true,
+                "ignoreTemplateLiterals": true
+            }
+        ],
+        "max-params": [ 0, 3 ],
+        "max-statements": [ 0, 10 ],
         "new-cap": [ "error",
             {
+                "newIsCap": true,
                 "capIsNew": false
             }
         ],
-        "no-alert": "error",
+        "new-parens": 2,
+        "newline-per-chained-call": 0,
+        "no-alert": 2,
         "no-array-constructor": "error",
-        "no-bitwise": "error",
-        "no-caller": "error",
-        "no-catch-shadow": "error",
-        "no-constant-condition": ["error", {"checkLoops": false}],
-        "no-div-regex": "error",
+        "no-bitwise": 2,
+        "no-caller": 2,
+        "no-case-declarations": "error",
+        "no-catch-shadow": 2,
+        "no-console": 0,
+        "no-const-assign": "error",
+        "no-control-regex": 2,
+        "no-debugger": 2,
+        "no-delete-var": 2,
+        "no-div-regex": 2,
+        "no-dupe-class-members": "error",
+        "no-dupe-keys": 2,
         "no-duplicate-imports": "error",
-        "no-else-return": "error",
-        "no-eq-null": "error",
-        "no-eval": "error",
-        "no-implied-eval": "error",
-        "no-inner-declarations": "off",
+        "no-else-return": 2,
+        "no-empty": 2,
+        "no-empty-character-class": 2,
+        "no-eq-null": 2,
+        "no-eval": 2,
+        "no-ex-assign": 2,
+        "no-extra-parens": [ "error", "functions" ],
+        "no-extra-semi": 2,
+        "no-fallthrough": 2,
+        "no-floating-decimal": 2,
+        "no-func-assign": 2,
+        "no-implied-eval": 2,
         "no-iterator": "error",
-        "no-label-var": "error",
-        "no-labels": "error",
-        "no-loop-func": "error",
-        "no-multi-str": "error",
-        "no-native-reassign": "error",
+        "no-label-var": 2,
+        "no-labels": 2,
+        "no-loop-func": 2,
+        "no-mixed-requires": [ 0, false ],
+        "no-multi-str": 2,
+        "no-native-reassign": 2,
+        "no-nested-ternary": 0,
         "no-new-func": "error",
-        "no-new-object": "error",
-        "no-new-wrappers": "error",
-        "no-octal-escape": "error",
-        "no-plusplus": "error",
-        "no-proto": "error",
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-self-compare": "error",
-        "no-sync": "error",
-        "no-undef-init": "error",
+        "no-new-object": 2,
+        "no-new-wrappers": 2,
+        "no-obj-calls": 2,
+        "no-octal": 2,
+        "no-octal-escape": 2,
+        "no-param-reassign": 0,
+        "no-plusplus": 2,
+        "no-proto": 2,
+        "no-redeclare": 2,
+        "no-regex-spaces": 2,
+        "no-restricted-syntax": 0,
+        "no-return-assign": 2,
+        "no-script-url": 2,
+        "no-self-compare": 2,
+        "no-shadow": 0,
+        "no-sync": 2,
+        "no-ternary": 0,
+        "no-undef": "error",
+        "no-undef-init": 2,
+        "no-underscore-dangle": 0,
         "no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
-        "no-unused-expressions": "error",
+        "no-unreachable": 2,
+        "no-unused-expressions": 2,
         "no-unused-vars": [ "error",
             {
                 "vars": "local",
+                "args": "after-used",
                 "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
             }
         ],
-        "no-use-before-define": "error",
+        "no-use-before-define": 2,
         "no-useless-constructor": "error",
-        "no-var": "error",
+        // The Zulip codebase complies partially with the "no-useless-escape"
+        // rule; only regex expressions haven't been updated yet.
+        // Updated regex expressions are currently being tested in casper
+        // files and will decide about a potential future enforcement of this rule.
+        "no-useless-escape": 0,
+        "no-whitespace-before-property": 0,
+        "no-with": 2,
         "one-var": [ "error", "never" ],
-        "prefer-arrow-callback": "error",
+        "padded-blocks": 0,
         "prefer-const": [ "error",
             {
+                "destructuring": "any",
                 "ignoreReadBeforeAssign": true
             }
         ],
-        "radix": "error",
-        "sort-imports": "error",
-        "spaced-comment": "off",
-        "strict": "off",
+        "quote-props": [ "error", "as-needed",
+            {
+                "keywords": false,
+                "unnecessary": true,
+                "numbers": false
+            }
+        ],
+        "quotes": [ 0, "single" ],
+        "radix": 2,
+        "semi": 2,
+        "space-before-blocks": 2,
+        "space-before-function-paren": [ "error",
+            {
+                "anonymous": "always",
+                "named": "never",
+                "asyncArrow": "always"
+            }
+        ],
+        "space-in-parens": 2,
+        "space-infix-ops": 0,
+        "spaced-comment": 0,
+        "strict": 0,
+        "template-curly-spacing": "error",
+        "unnecessary-strict": 0,
+        "use-isnan": 2,
         "valid-typeof": [ "error", { "requireStringLiterals": true } ],
-        "yoda": "error"
-    },
-    "overrides": [
-        {
-            "files": [
-                "frontend_tests/**/*.{js,ts}",
-                "static/js/**/*.{js,ts}"
-            ],
-            "globals": {
-                "$": false,
-                "ClipboardJS": false,
-                "FetchStatus": false,
-                "Filter": false,
-                "Handlebars": false,
-                "LightboxCanvas": false,
-                "MessageListData": false,
-                "MessageListView": false,
-                "Plotly": false,
-                "Sortable": false,
-                "WinChan": false,
-                "XDate": false,
-                "_": false,
-                "activity": false,
-                "admin": false,
-                "alert_words": false,
-                "alert_words_ui": false,
-                "attachments_ui": false,
-                "avatar": false,
-                "billing": false,
-                "blueslip": false,
-                "bot_data": false,
-                "bridge": false,
-                "buddy_data": false,
-                "buddy_list": false,
-                "channel": false,
-                "click_handlers": false,
-                "color_data": false,
-                "colorspace": false,
-                "common": false,
-                "components": false,
-                "compose": false,
-                "compose_actions": false,
-                "compose_fade": false,
-                "compose_pm_pill": false,
-                "compose_state": false,
-                "compose_ui": false,
-                "composebox_typeahead": false,
-                "condense": false,
-                "confirm_dialog": false,
-                "copy_and_paste": false,
-                "csrf_token": false,
-                "current_msg_list": true,
-                "drafts": false,
-                "dropdown_list_widget": false,
-                "echo": false,
-                "emoji": false,
-                "emoji_picker": false,
-                "favicon": false,
-                "feature_flags": false,
-                "feedback_widget": false,
-                "fenced_code": false,
-                "flatpickr": false,
-                "floating_recipient_bar": false,
-                "gear_menu": false,
-                "hash_util": false,
-                "hashchange": false,
-                "helpers": false,
-                "history": false,
-                "home_msg_list": false,
-                "hotspots": false,
-                "i18n": false,
-                "info_overlay": false,
-                "input_pill": false,
-                "invite": false,
-                "jQuery": false,
-                "katex": false,
-                "keydown_util": false,
-                "lightbox": false,
-                "list_cursor": false,
-                "list_render": false,
-                "list_util": false,
-                "loading": false,
-                "localStorage": false,
-                "local_message": false,
-                "localstorage": false,
-                "location": false,
-                "markdown": false,
-                "marked": false,
-                "md5": false,
-                "message_edit": false,
-                "message_edit_history": false,
-                "message_events": false,
-                "message_fetch": false,
-                "message_flags": false,
-                "message_list": false,
-                "message_live_update": false,
-                "message_scroll": false,
-                "message_store": false,
-                "message_util": false,
-                "message_viewport": false,
-                "moment": false,
-                "muting": false,
-                "muting_ui": false,
-                "narrow": false,
-                "narrow_state": false,
-                "navigate": false,
-                "night_mode": false,
-                "notifications": false,
-                "overlays": false,
-                "padded_widget": false,
-                "page_params": false,
-                "panels": false,
-                "people": false,
-                "pm_conversations": false,
-                "pm_list": false,
-                "pm_list_dom": false,
-                "pointer": false,
-                "popovers": false,
-                "presence": false,
-                "reactions": false,
-                "realm_icon": false,
-                "realm_logo": false,
-                "realm_night_logo": false,
-                "recent_senders": false,
-                "recent_topics": false,
-                "reload": false,
-                "reload_state": false,
-                "reminder": false,
-                "resize": false,
-                "rows": false,
-                "rtl": false,
-                "run_test": false,
-                "schema": false,
-                "scroll_bar": false,
-                "scroll_util": false,
-                "search": false,
-                "search_pill": false,
-                "search_pill_widget": false,
-                "search_suggestion": false,
-                "search_util": false,
-                "sent_messages": false,
-                "server_events": false,
-                "server_events_dispatch": false,
-                "settings": false,
-                "settings_account": false,
-                "settings_bots": false,
-                "settings_display": false,
-                "settings_emoji": false,
-                "settings_exports": false,
-                "settings_linkifiers": false,
-                "settings_invites": false,
-                "settings_muting": false,
-                "settings_notifications": false,
-                "settings_org": false,
-                "settings_panel_menu": false,
-                "settings_profile_fields": false,
-                "settings_sections": false,
-                "settings_streams": false,
-                "settings_toggle": false,
-                "settings_ui": false,
-                "settings_user_groups": false,
-                "settings_users": false,
-                "spoilers": false,
-                "starred_messages": false,
-                "stream_color": false,
-                "stream_create": false,
-                "stream_data": false,
-                "stream_edit": false,
-                "stream_events": false,
-                "stream_topic_history": false,
-                "stream_list": false,
-                "stream_muting": false,
-                "stream_popover": false,
-                "stream_sort": false,
-                "stream_ui_updates": false,
-                "StripeCheckout": false,
-                "submessage": false,
-                "subs": false,
-                "tab_bar": false,
-                "templates": false,
-                "tictactoe_widget": false,
-                "timerender": false,
-                "todo_widget": false,
-                "top_left_corner": false,
-                "topic_generator": false,
-                "topic_list": false,
-                "topic_zoom": false,
-                "transmit": false,
-                "tutorial": false,
-                "typeahead_helper": false,
-                "typing": false,
-                "typing_data": false,
-                "typing_events": false,
-                "ui": false,
-                "ui_init": false,
-                "ui_report": false,
-                "ui_util": false,
-                "unread": false,
-                "unread_ops": false,
-                "unread_ui": false,
-                "upgrade": false,
-                "upload": false,
-                "upload_widget": false,
-                "user_events": false,
-                "user_groups": false,
-                "user_pill": false,
-                "user_search": false,
-                "user_status": false,
-                "user_status_ui": false,
-                "poll_widget": false,
-                "vdom": false,
-                "widgetize": false,
-                "zcommand": false,
-                "zform": false,
-                "zxcvbn": false
-            }
-        },
-        {
-            "files": [
-                "frontend_tests/casper_tests/*.js",
-                "frontend_tests/casper_lib/*.js"
-            ],
-            "rules": {
-                // Don’t require ES features that PhantomJS doesn’t support
-                "no-var": "off",
-                "prefer-arrow-callback": "off"
-            }
-        },
-        {
-            "files": ["**/*.ts"],
-            "extends": [
-                "plugin:@typescript-eslint/recommended",
-                "prettier/@typescript-eslint"
-            ],
-            "parserOptions": {
-                "project": "tsconfig.json"
-            },
-            "rules": {
-                // Disable base rule to avoid conflict
-                "empty-returns/main": "off",
-                "no-unused-vars": "off",
-                "no-useless-constructor": "off",
-
-                "@typescript-eslint/array-type": "error",
-                "@typescript-eslint/await-thenable": "error",
-                "@typescript-eslint/consistent-type-assertions": "error",
-                "@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
-                "@typescript-eslint/member-ordering": "error",
-                "@typescript-eslint/no-explicit-any": "off",
-                "@typescript-eslint/no-extraneous-class": "error",
-                "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-parameter-properties": "error",
-                "@typescript-eslint/no-unnecessary-qualifier": "error",
-                "@typescript-eslint/no-unnecessary-type-assertion": "error",
-                "@typescript-eslint/no-unused-vars": ["error", { "varsIgnorePattern": "^_" } ],
-                "@typescript-eslint/no-use-before-define": "error",
-                "@typescript-eslint/no-useless-constructor": "error",
-                "@typescript-eslint/prefer-includes": "error",
-                "@typescript-eslint/prefer-regexp-exec": "error",
-                "@typescript-eslint/prefer-string-starts-ends-with": "error",
-                "@typescript-eslint/promise-function-async": "error",
-                "@typescript-eslint/unified-signatures": "error"
-            }
-        }
-    ]
+        "wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
+        "wrap-regex": 0,
+        "yoda": 2
+    }
 }

.gitattributes

@@ -1,7 +1,6 @@
 *   text=auto eol=lf
 *.gif binary
 *.jpg binary
-*.jpeg binary
 *.eot binary
 *.woff binary
 *.woff2 binary
@@ -10,4 +9,3 @@
 *.png binary
 *.otf binary
 *.tif binary
-*.ogg binary

.github/pull_request_template.md

@@ -1,14 +0,0 @@
-<!-- What's this PR for?  (Just a link to an issue is fine.) -->
-
-
-**Testing Plan:** <!-- How have you tested? -->
-
-
-**GIFs or Screenshots:** <!-- If a UI change.  See:
-  https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
-  -->
-
-
-<!-- Also be sure to make clear, coherent commits:
-  https://zulip.readthedocs.io/en/latest/contributing/version-control.html
-  -->

.github/workflows/codeql-analysis.yml

@@ -1,30 +0,0 @@
-name: "Code Scanning"
-
-on: [push, pull_request]
-
-jobs:
-  CodeQL:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          # We must fetch at least the immediate parents so that if this is
-          # a pull request then we can checkout the head.
-          fetch-depth: 2
-
-      # If this run was triggered by a pull request event, then checkout
-      # the head of the pull request instead of the merge commit.
-      - run: git checkout HEAD^2
-        if: ${{ github.event_name == 'pull_request' }}
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
-
-        # Override language selection by uncommenting this and choosing your languages
-        # with:
-        #   languages: go, javascript, csharp, python, cpp, java
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1

.github/workflows/zulip-ci.yml

@@ -1,171 +0,0 @@
-name: Zulip CI
-
-on: [push, pull_request]
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  focal_bionic:
-    strategy:
-      matrix:
-        include:
-          # This docker image was created by a generated Dockerfile at:
-          #   tools/ci/images/bionic/Dockerfile
-          # Bionic ships with Python 3.6.
-          - docker_image: mepriyank/actions:bionic
-            name: Ubuntu 18.04 Bionic (Python 3.6, backend + frontend)
-            os: bionic
-            is_bionic: true
-            include_frontend_tests: true
-
-          # This docker image was created by a generated Dockerfile at:
-          #   tools/ci/images/focal/Dockerfile
-          # Focal ships with Python 3.8.2.
-          - docker_image: mepriyank/actions:focal
-            name: Ubuntu 20.04 Focal (Python 3.8, backend)
-            os: focal
-            is_focal: true
-            include_frontend_tests: false
-
-    runs-on: ubuntu-latest
-    name: ${{ matrix.name }}
-    container: ${{ matrix.docker_image }}
-    env:
-      # GitHub Actions sets HOME to /github/home which causes
-      # problem later in provison and frontend test that runs
-      # tools/setup/postgres-init-dev-db because of the .pgpass
-      # location. Postgresql (psql) expects .pgpass to be at
-      # /home/github/.pgpass and setting home to `/home/github/`
-      # ensures it written there because we write it to ~/.pgpass.
-      HOME: /home/github/
-
-    steps:
-      - name: Add required permissions
-        run: |
-          # The checkout actions doesn't clone to ~/zulip or allow
-          # us to use the path option to clone outside the current
-          # /__w/zulip/zulip directory. Since this directory is owned
-          # by root we need to change it's ownership to allow the
-          # github user to clone the code here.
-          # Note: /__w/ is a docker volume mounted to $GITHUB_WORKSPACE
-          # which is /home/runner/work/.
-          sudo chown -R github .
-
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-      - uses: actions/checkout@v2
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore node_modules cache
-        uses: actions/cache@v2
-        with:
-          path: /srv/zulip-npm-cache
-          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('package.json') }}-${{ hashFiles('yarn.lock') }}
-          restore-keys: v1-yarn-deps-${{ matrix.os }}
-
-      - name: Restore python cache
-        uses: actions/cache@v2
-        with:
-          path: /srv/zulip-venv-cache
-          key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/thumbor-dev.txt') }}-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-${{ matrix.os }}
-
-      - name: Restore emoji cache
-        uses: actions/cache@v2
-        with:
-          path: /srv/zulip-emoji-cache
-          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
-          restore-keys: v1-emoji-${{ matrix.os }}
-
-      - name: Do Bionic hack
-        if: ${{ matrix.is_bionic }}
-        run: |
-          # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
-          # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
-          sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
-
-      - name: Install dependencies
-        run: |
-          # This is the main setup job for the test suite
-          mispipe "tools/ci/setup-backend --skip-dev-db-build" ts
-
-          # Cleaning caches is mostly unnecessary in GitHub Actions, because
-          # most builds don't get to write to the cache.
-          # mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0 2>&1" ts
-
-      - name: Run backend tests
-        run: |
-          . /srv/zulip-py3-venv/bin/activate && \
-          mispipe "./tools/ci/backend 2>&1" ts
-
-      - name: Run frontend tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          . /srv/zulip-py3-venv/bin/activate
-          mispipe "./tools/ci/frontend 2>&1" ts
-
-      - name: Test locked requirements
-        if: ${{ matrix.is_bionic }}
-        run: |
-          . /srv/zulip-py3-venv/bin/activate && \
-          mispipe "./tools/test-locked-requirements 2>&1" ts
-
-      - name: Upload coverage reports
-
-        # Only upload coverage when both frontend and backend
-        # tests are ran.
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          # Codcov requires `.coverage` file to be stored in the
-          # current working directory.
-          mv ./var/.coverage ./.coverage
-          . /srv/zulip-py3-venv/bin/activate || true
-
-          # TODO: Check that the next release of codecov doesn't
-          # throw find error.
-          # codecov==2.0.16 introduced a bug which uses "find"
-          # for locating files which is buggy on some platforms.
-          # It was fixed via https://github.com/codecov/codecov-python/pull/217
-          # and should get automatically fixed here once it's released.
-          # We cannot pin the version here because we need the latest version for uploading files.
-          # see https://community.codecov.io/t/http-400-while-uploading-to-s3-with-python-codecov-from-travis/1428/7
-          pip install codecov && codecov || echo "Error in uploading coverage reports to codecov.io."
-
-      - name: Store puppeteer artifacts
-        if: ${{ matrix.include_frontend_tests }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: puppeteer
-          path: ./var/puppeteer
-
-      # We cannot use upload-artifacts actions to upload the test
-      # reports from /tmp, that directory exists inside the docker
-      # image. Move them to ./var so we access it outside docker since
-      # the current directory is volume mounted outside the docker image.
-      - name: Move test reports to var
-        run: mv /tmp/zulip-test-event-log/ ./var/
-
-      - name: Store test reports
-        if: ${{ matrix.is_bionic }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: test-reports
-          path: ./var/zulip-test-event-log/
-
-      - name: Check development database build
-        if: ${{ matrix.is_focal }}
-        run: mispipe "tools/ci/setup-backend" ts
-      # TODO: We need to port the notify_failure step from CircleCI
-      # config, however, it might be the case that GitHub Notifications
-      # make this unnesscary. More details on settings to configure it:
-      # https://help.github.com/en/github/managing-subscriptions-and-notifications-on-github/configuring-notifications#github-actions-notification-options

.gitignore

@@ -12,34 +12,19 @@
 #  * Subdirectories with several internal things to ignore get their own
 #    `.gitignore` files.
 #
-#  * Comments must be on their own line.  (Otherwise they don't work.)
-#
 # See `git help ignore` for details on the format.
 
 ## Config files for the dev environment
 /zproject/dev-secrets.conf
 /tools/conf.ini
 /tools/custom_provision
-/tools/droplets/conf.ini
 
 ## Byproducts of setting up and using the dev environment
 *.pyc
-package-lock.json
 
 /.vagrant
 /var
 
-/.dmypy.json
-
-# Dockerfiles generated for CircleCI
-/tools/ci/images
-
-# Generated i18n data
-/locale/en
-/locale/language_options.json
-/locale/language_name_map.json
-/locale/*/mobile.json
-
 # Static build
 *.mo
 npm-debug.log
@@ -48,16 +33,10 @@ npm-debug.log
 /staticfiles.json
 /webpack-stats-production.json
 /yarn-error.log
-zulip-git-version
 
 # Test / analysis tools
 .coverage
 
-## Files (or really symlinks) created in a prod deployment
-/zproject/prod_settings.py
-/zulip-current-venv
-/zulip-py3-venv
-
 ## Files left by various editors and local environments
 # (Ideally these should be in everyone's respective personal gitignore files.)
 *~
@@ -72,14 +51,6 @@ zulip.kdev4
 *.sublime-workspace
 .vscode/
 *.DS_Store
-# .cache/ is generated by VSCode's test runner
-.cache/
-.eslintcache
-
-# Core dump files
-core
 
 ## Miscellaneous
 # (Ideally this section is empty.)
-zthumbor/thumbor_local_settings.py
-.transifexrc

.gitlint

@@ -1,10 +1,10 @@
 [general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing, title-imperative-mood
+ignore=title-trailing-punctuation, body-min-length, body-is-missing
 
 extra-path=tools/lib/gitlint-rules.py
 
-[title-match-regex-allow-exception]
-regex=^(.+:\ )?[A-Z].+\.$
+[title-match-regex]
+regex=^.+\.$
 
 [title-max-length]
 line-length=76

.isort.cfg

@@ -1,7 +0,0 @@
-[settings]
-src_paths = ., tools, tools/setup/emoji
-multi_line_output = 3
-known_third_party = zulip
-include_trailing_comma = True
-use_parentheses = True
-line_length = 100

.mailmap

@@ -1,29 +0,0 @@
-Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
-Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
-Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
-Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
-Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
-Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
-Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
-Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
-Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-Greg Price <greg@zulip.com> <gnprice@gmail.com>
-Greg Price <greg@zulip.com> <greg@zulipchat.com>
-Greg Price <greg@zulip.com> <price@mit.edu>
-Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
-Rishi Gupta <rishig@zulip.com> <rishig+git@mit.edu>
-Rishi Gupta <rishig@zulip.com> <rishig@kandralabs.com>
-Rishi Gupta <rishig@zulip.com> <rishig@users.noreply.github.com>
-Rishi Gupta <rishig@zulip.com> <rishig@zulipchat.com>
-Steve Howell <showell@zulip.com> <showell30@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@zulipchat.com>
-Steve Howell <showell@zulip.com> <steve@humbughq.com>
-Steve Howell <showell@zulip.com> <steve@zulip.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
-Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Vishnu KS <yo@vishnuks.com> <hackerkid@vishnuks.com>
-Vishnu KS <yo@vishnuks.com> <yo@vishnuks.com>

.prettierignore

@@ -1 +0,0 @@
-/static/third

.pyre_configuration

@@ -1,14 +0,0 @@
-{
-  "source_directories": ["."],
-  "taint_models_path": [
-      "stubs/taint",
-      "zulip-py3-venv/lib/pyre_check/taint/"
-  ],
-  "search_path": [
-      "stubs/",
-      "zulip-py3-venv/lib/pyre_check/stubs/"
-  ],
-  "exclude": [
-      "/srv/zulip/zulip-py3-venv/.*"
-  ]
-}

.sonarcloud.properties

@@ -1 +0,0 @@
-sonar.inclusions=**/*.py,**/*.html

.stylelintrc

@@ -1,67 +0,0 @@
-{
-    "rules": {
-        # Stylistic rules for CSS.
-        "function-comma-space-after": "always",
-        "function-comma-space-before": "never",
-        "function-max-empty-lines": 0,
-        "function-whitespace-after": "always",
-
-        "value-keyword-case": "lower",
-        "value-list-comma-newline-after": "always-multi-line",
-        "value-list-comma-space-after": "always-single-line",
-        "value-list-comma-space-before": "never",
-        "value-list-max-empty-lines": 0,
-
-        "unit-case": "lower",
-        "property-case": "lower",
-        "color-hex-case": "lower",
-
-        "declaration-bang-space-before": "always",
-        "declaration-colon-newline-after": "always-multi-line",
-        "declaration-colon-space-after": "always-single-line",
-        "declaration-colon-space-before": "never",
-        "declaration-block-semicolon-newline-after": "always",
-        "declaration-block-semicolon-space-before": "never",
-        "declaration-block-trailing-semicolon": "always",
-
-        "block-closing-brace-empty-line-before": "never",
-        "block-closing-brace-newline-after": "always",
-        "block-closing-brace-newline-before": "always",
-        "block-opening-brace-newline-after": "always",
-        "block-opening-brace-space-before": "always",
-
-        "selector-attribute-brackets-space-inside": "never",
-        "selector-attribute-operator-space-after": "never",
-        "selector-attribute-operator-space-before": "never",
-        "selector-combinator-space-after": "always",
-        "selector-combinator-space-before": "always",
-        "selector-descendant-combinator-no-non-space": true,
-        "selector-pseudo-class-parentheses-space-inside": "never",
-        "selector-pseudo-element-case": "lower",
-        "selector-pseudo-element-colon-notation": "double",
-        "selector-type-case": "lower",
-        "selector-list-comma-newline-after": "always",
-        "selector-list-comma-space-before": "never",
-
-        "media-feature-colon-space-after": "always",
-        "media-feature-colon-space-before": "never",
-        "media-feature-name-case": "lower",
-        "media-feature-parentheses-space-inside": "never",
-        "media-feature-range-operator-space-after": "always",
-        "media-feature-range-operator-space-before": "always",
-        "media-query-list-comma-newline-after": "always",
-        "media-query-list-comma-space-before": "never",
-
-        "at-rule-name-case": "lower",
-        "at-rule-name-space-after": "always",
-        "at-rule-semicolon-newline-after": "always",
-        "at-rule-semicolon-space-before": "never",
-
-        "comment-whitespace-inside": "always",
-        "indentation": 4,
-        
-        # Limit language features
-        "color-no-hex": true,
-        "color-named": "never",
-    }
-}

.travis.yml

@@ -0,0 +1,75 @@
+# See https://zulip.readthedocs.io/en/latest/travis.html for
+# high-level documentation on our Travis CI setup.
+dist: trusty
+install:
+  # Disable broken riak sources.list in Travis base image 2017-10-18
+  - rm -vf "/etc/apt/sources.list.d/*riak*"
+
+  # Disable Travis CI's built-in NVM installation
+  - mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
+
+  # Install codecov, the library for the code coverage reporting tool we use
+  # With a retry to minimize impact of transient networking errors.
+  - mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
+
+  # This is the main setup job for the test suite
+  - mispipe "tools/travis/setup-$TEST_SUITE" ts
+
+  # Clean any caches that are not in use to avoid our cache
+  # becoming huge.
+  - mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
+
+script:
+  # We unset GEM_PATH here as a hack to work around Travis CI having
+  # broken running their system puppet with Ruby.  See
+  # https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
+  - unset GEM_PATH
+  - mispipe "./tools/travis/$TEST_SUITE" ts
+cache:
+  yarn: true
+  apt: false
+  directories:
+    - $HOME/zulip-venv-cache
+    - $HOME/zulip-npm-cache
+    - $HOME/zulip-emoji-cache
+    - $HOME/node
+env:
+  global:
+    - BOTO_CONFIG=/tmp/nowhere
+language: python
+# Our test suites generally run on Python 3.4, the version in
+# Ubuntu 14.04 trusty, which is the oldest OS release we support.
+matrix:
+  include:
+    # Travis will actually run the jobs in the order they're listed here;
+    # that doesn't seem to be documented, but it's what we see empirically.
+    # We only get 4 jobs running at a time, so we try to make the first few
+    # the most likely to break.
+    - python: "3.4"
+      env: TEST_SUITE=frontend
+    - python: "3.4"
+      env: TEST_SUITE=backend
+    - python: "3.4"
+      env: TEST_SUITE=production
+    - python: "3.5"
+      env: TEST_SUITE=backend
+sudo: required
+addons:
+  artifacts:
+    paths:
+      # Casper debugging data (screenshots, etc.) is super useful for
+      # debugging test flakes.
+      - $(ls var/casper/* | tr "\n" ":")
+      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
+  postgresql: "9.3"
+  apt:
+    packages:
+      - moreutils
+after_success:
+  - codecov
+notifications:
+  webhooks:
+    urls:
+      - https://zulip.org/zulipbot/travis
+    on_success: always
+    on_failure: always

.tx/config

@@ -3,31 +3,13 @@ host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
 [zulip.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
+source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
+file_filter = static/locale/<lang>/LC_MESSAGES/django.po
 
 [zulip.translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[zulip.mobile]
-file_filter = locale/<lang>/mobile.json
-source_file = locale/en/mobile.json
-source_lang = en
-type = KEYVALUEJSON
-
-[zulip-test.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
-source_lang = en
-type = PO
-
-[zulip-test.translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
+source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
+file_filter = static/locale/<lang>/translations.json

.yarnrc

@@ -1 +0,0 @@
-ignore-scripts true

CODE_OF_CONDUCT.md

@@ -78,7 +78,7 @@ something you can do while a violation is happening, do it. A lot of the
 harms of harassment and other violations can be mitigated by the victim
 knowing that the other people present are on their side.
 
-All reports will be kept confidential. In some cases, we may determine that a
+All reports will be kept confidential. In some cases we may determine that a
 public statement will need to be made. In such cases, the identities of all
 victims and reporters will remain confidential unless those individuals
 instruct us otherwise.
@@ -101,5 +101,5 @@ This Code of Conduct is adapted from the
 [Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
 [Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
 under a
-[Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
+[Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
 license.

CONTRIBUTING.md

@@ -1,340 +0,0 @@
-# Contributing to Zulip
-
-Welcome to the Zulip community!
-
-## Community
-
-The
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-is the primary communication forum for the Zulip community. It is a good
-place to start whether you have a question, are a new contributor, are a new
-user, or anything else. Make sure to read the
-[community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
-before posting. The Zulip community is also governed by a
-[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
-
-You can subscribe to zulip-devel-announce@googlegroups.com or our
-[Twitter](https://twitter.com/zulip) account for a lower traffic (~1
-email/month) way to hear about things like mentorship opportunities with Google
-Code-in, in-person sprints at conferences, and other opportunities to
-contribute.
-
-## Ways to contribute
-
-To make a code or documentation contribution, read our
-[step-by-step guide](#your-first-codebase-contribution) to getting
-started with the Zulip codebase. A small sample of the type of work that
-needs doing:
-* Bug squashing and feature development on our Python/Django
-  [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip), React Native
-  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
-  [desktop app](https://github.com/zulip/zulip-desktop).
-* Building out our
-  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
-* [Writing an integration](https://zulip.com/api/integrations-overview).
-* Improving our [user](https://zulip.com/help/) or
-  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
-* [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
-  and manually testing pull requests.
-
-**Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. We list a few of them below:
-
-* [Reporting issues](#reporting-issues), including both feature requests and
-  bug reports.
-* [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
-* [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip.
-* [Outreach](#zulip-outreach): Star us on GitHub, upvote us
-  on product comparison sites, or write for [the Zulip blog](https://blog.zulip.org/).
-
-## Your first (codebase) contribution
-
-This section has a step by step guide to starting as a Zulip codebase
-contributor. It's long, but don't worry about doing all the steps perfectly;
-no one gets it right the first time, and there are a lot of people available
-to help.
-* First, make an account on the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
-  paying special attention to the community norms. If you'd like, introduce
-  yourself in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
-  your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken as you started using the
-  product.
-* Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-* [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-  getting help in
-  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
-  if you run into any troubles.
-* Read the
-  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
-  and do the Git tutorial (coming soon) if you are unfamiliar with
-  Git, getting help in
-  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
-  you run into any troubles.  Be sure to check out the
-  [extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
-
-### Picking an issue
-
-Now, you're ready to pick your first issue! There are hundreds of open issues
-in the main codebase alone. This section will help you find an issue to work
-on.
-
-* If you're interested in
-  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
-  [desktop](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue),
-  or
-  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
-  development, check the respective links for open issues, or post in
-  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
-  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
-  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
-* For the main server and web repository, we recommend browsing
-  recently opened issues to look for issues you are confident you can
-  fix correctly in a way that clearly communicates why your changes
-  are the correct fix.  Our GitHub workflow bot, zulipbot, limits
-  users who have 0 commits merged to claiming a single issue labeled
-  with "good first issue" or "help wanted".
-* We also partition all of our issues in the main repo into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-* If the lists of issues are overwhelming, post in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
-  bit about your background and interests, and we'll help you out. The most
-  important thing to say is whether you're looking for a backend (Python),
-  frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
-  documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
-  bit about your programming experience and available time.
-
-We also welcome suggestions of features that you feel would be valuable or
-changes that you feel would make Zulip a better open source project. If you
-have a new feature you'd like to add, we recommend you start by posting in
-[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
-feature idea and the problem that you're hoping to solve.
-
-Other notes:
-* For a first pull request, it's better to aim for a smaller contribution
-  than a bigger one. Many first contributions have fewer than 10 lines of
-  changes (not counting changes to tests).
-* The full list of issues explicitly looking for a contributor can be
-  found with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  and
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels.  Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are confident you can resolve the
-  issue correctly and completely.  Issues without one of these labels
-  are fair game if Tim has written a clear technical design proposal
-  in the issue, or it is a bug that you can reproduce and you are
-  confident you can fix the issue correctly.
-* For most new contributors, there's a lot to learn while making your first
-  pull request. It's OK if it takes you a while; that's normal! You'll be
-  able to work a lot faster as you build experience.
-
-### Working on an issue
-
-To work on an issue, claim it by adding a comment with `@zulipbot claim` to
-the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
-workflow bot; it will assign you to the issue and label the issue as "in
-progress". Some additional notes:
-
-* You can only claim issues with the
-  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-  or
-  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-  labels. Zulipbot will give you an error if you try to claim an issue
-  without one of those labels.
-* You're encouraged to ask questions on how to best implement or debug your
-  changes -- the Zulip maintainers are excited to answer questions to help
-  you stay unblocked and working efficiently. You can ask questions on
-  chat.zulip.org, or on the GitHub issue or pull request.
-* We encourage early pull requests for work in progress. Prefix the title of
-  work in progress pull requests with `[WIP]`, and remove the prefix when
-  you think it might be mergeable and want it to be reviewed.
-* After updating a PR, add a comment to the GitHub thread mentioning that it
-  is ready for another review. GitHub only notifies maintainers of the
-  changes when you post a comment, so if you don't, your PR will likely be
-  neglected by accident!
-
-### And beyond
-
-A great place to look for a second issue is to look for issues with the same
-`area:` label as the last issue you resolved. You'll be able to reuse the
-work you did learning how that part of the codebase works. Also, the path to
-becoming a core developer often involves taking ownership of one of these area
-labels.
-
-## What makes a great Zulip contributor?
-
-Zulip has a lot of experience working with new contributors.  In our
-experience, these are the best predictors of success:
-
-* Posting good questions. This generally means explaining your current
-  understanding, saying what you've done or tried so far, and including
-  tracebacks or other error messages if appropriate.
-* Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
-* Submitting carefully tested code. This generally means checking your work
-  through a combination of automated tests and manually clicking around the
-  UI trying to find bugs in your work. See
-  [things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
-  for additional ideas.
-* Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-* Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-* Being helpful and friendly on chat.zulip.org.
-
-These are also the main criteria we use to select candidates for all
-of our outreach programs.
-
-## Reporting issues
-
-If you find an easily reproducible bug and/or are experienced in reporting
-bugs, feel free to just open an issue on the relevant project on GitHub.
-
-If you have a feature request or are not yet sure what the underlying bug
-is, the best place to post issues is
-[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
-[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
-[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
-This allows us to interactively figure out what is going on, let you know if
-a similar issue has already been opened, and collect any other information
-we need. Choose a 2-4 word topic that describes the issue, explain the issue
-and how to reproduce it if known, your browser/OS if relevant, and a
-[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-if appropriate.
-
-**Reporting security issues**. Please do not report security issues
-  publicly, including on public streams on chat.zulip.org. You can
-  email security@zulip.com.  We create a CVE for every security
-  issue in our released software.
-
-## User feedback
-
-Nearly every feature we develop starts with a user request. If you are part
-of a group that is either using or considering using Zulip, we would love to
-hear about your experience with the product. If you're not sure what to
-write, here are some questions we're always very curious to know the answer
-to:
-
-* Evaluation: What is the process by which your organization chose or will
-  choose a group chat product?
-* Pros and cons: What are the pros and cons of Zulip for your organization,
-  and the pros and cons of other products you are evaluating?
-* Features: What are the features that are most important for your
-  organization? In the best-case scenario, what would your chat solution do
-  for you?
-* Onboarding: If you remember it, what was your impression during your first
-  few minutes of using Zulip? What did you notice, and how did you feel? Was
-  there anything that stood out to you as confusing, or broken, or great?
-* Organization: What does your organization do? How big is the organization?
-  A link to your organization's website?
-
-## Outreach programs
-
-Zulip participates in [Google Summer of Code
-(GSoC)](https://developers.google.com/open-source/gsoc/) every year.
-In the past, we've also participated in
-[Outreachy](https://www.outreachy.org/), [Google
-Code-In](https://developers.google.com/open-source/gci/), and hosted
-summer interns from Harvard, MIT, and Stanford.
-
-While each third-party program has its own rules and requirements, the
-Zulip community's approaches all of these programs with these ideas in
-mind:
-* We try to make the application process as valuable for the applicant as
-  possible. Expect high-quality code reviews, a supportive community, and
-  publicly viewable patches you can link to from your resume, regardless of
-  whether you are selected.
-* To apply, you'll have to submit at least one pull request to a Zulip
-  repository.  Most students accepted to one of our programs have
-  several merged pull requests (including at least one larger PR) by
-  the time of the application deadline.
-* The main criteria we use is quality of your best contributions, and
-  the bullets listed at
-  [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-  Because we focus on evaluating your best work, it doesn't hurt your
-  application to makes mistakes in your first few PRs as long as your
-  work improves.
-
-Most of our outreach program participants end up sticking around the
-project long-term, and many have become core team members, maintaining
-important parts of the project. We hope you apply!
-
-### Google Summer of Code
-
-The largest outreach program Zulip participates in is GSoC (14
-students in 2017; 11 in 2018; 17 in 2019).  While we don't control how
-many slots Google allocates to Zulip, we hope to mentor a similar
-number of students in future summers.
-
-If you're reading this well before the application deadline and want
-to make your application strong, we recommend getting involved in the
-community and fixing issues in Zulip now. Having good contributions
-and building a reputation for doing good work is the best way to have
-a strong application.  About half of Zulip's GSoC students for Summer
-2017 had made significant contributions to the project by February
-2017, and about half had not.  Our
-[GSoC project ideas page][gsoc-guide] has lots more details on how
-Zulip does GSoC, as well as project ideas (though the project idea
-list is maintained only during the GSoC application period, so if
-you're looking at some other time of year, the project list is likely
-out-of-date).
-
-We also have in some past years run a Zulip Summer of Code (ZSoC)
-program for students who we didn't have enough slots to accept for
-GSoC but were able to find funding for.  Student expectations are the
-same as with GSoC, and it has no separate application process; your
-GSoC application is your ZSoC application.  If we'd like to select you
-for ZSoC, we'll contact you when the GSoC results are announced.
-
-[gsoc-guide]: https://zulip.readthedocs.io/en/latest/overview/gsoc-ideas.html
-[gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
-
-## Zulip Outreach
-
-**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
-perception of projects like Zulip. We've collected a few sites below
-where we know Zulip has been discussed. Doing everything in the following
-list typically takes about 15 minutes.
-* Star us on GitHub. There are four main repositories:
-  [server/web](https://github.com/zulip/zulip),
-  [mobile](https://github.com/zulip/zulip-mobile),
-  [desktop](https://github.com/zulip/zulip-desktop), and
-  [Python API](https://github.com/zulip/python-zulip-api).
-* [Follow us](https://twitter.com/zulip) on Twitter.
-
-For both of the following, you'll need to make an account on the site if you
-don't already have one.
-
-* [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
-  AlternativeTo. We recommend upvoting a couple of other products you like
-  as well, both to give back to their community, and since single-upvote
-  accounts are generally given less weight. You can also
-  [upvote Zulip](https://alternativeto.net/software/slack/) on their page
-  for Slack.
-* [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
-  it, and upvote the reasons why people like Zulip that you find most
-  compelling. Again, we recommend adding a few other products that you like
-  as well.
-
-We have a doc with more detailed instructions and a few other sites, if you
-have been using Zulip for a while and want to contribute more.
-
-**Blog posts**. Writing a blog post about your experiences with Zulip, or
-about a technical aspect of Zulip can be a great way to spread the word
-about Zulip.
-
-We also occasionally [publish](https://blog.zulip.org/) long-form
-articles related to Zulip. Our posts typically get tens of thousands
-of views, and we always have good ideas for blog posts that we can
-outline but don't have time to write. If you are an experienced writer
-or copyeditor, send us a portfolio; we'd love to talk!

Dockerfile-dev

@@ -0,0 +1,17 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y wget
+
+RUN locale-gen en_US.UTF-8
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
+
+WORKDIR /srv/zulip

Dockerfile-postgresql

@@ -1,15 +0,0 @@
-# To build run `docker build -f Dockerfile-postgresql .` from the root of the
-# zulip repo.
-
-# Currently the postgres images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image can not currently be upgraded
-# without users needing a manual pgdump and restore.
-
-# Install hunspell, zulip stop words, and run zulip database
-# init.
-FROM groonga/pgroonga:latest-alpine-10-slim
-RUN apk add -U --no-cache hunspell-en
-RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix 
-COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/local/share/postgresql/tsearch_data/zulip_english.stop
-COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
-COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql

LICENSE

@@ -1,4 +1,3 @@
-Copyright 2011-2020 Dropbox, Inc., Kandra Labs, Inc., and contributors
 
                                  Apache License
                            Version 2.0, January 2004

NOTICE

@@ -1,16 +0,0 @@
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this project except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-The software includes some works released by third parties under other
-free and open source licenses. Those works are redistributed under the
-license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.

README.md

@@ -1,77 +1,301 @@
+**[Zulip overview](#zulip-overview)** |
+**[Community](#community)** |
+**[Installing for dev](#installing-the-zulip-development-environment)** |
+**[Installing for production](#running-zulip-in-production)** |
+**[Ways to contribute](#ways-to-contribute)** |
+**[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
+**[License](#license)**
+
 # Zulip overview
 
-Zulip is a powerful, open source group chat application that combines the
-immediacy of real-time chat with the productivity benefits of threaded
-conversations. Zulip is used by open source projects, Fortune 500 companies,
-large standards bodies, and others who need a real-time chat system that
-allows users to easily process hundreds or thousands of messages a day. With
-over 500 contributors merging over 500 commits a month, Zulip is also the
-largest and fastest growing open source group chat project.
+Zulip is a powerful, open source group chat application. Written in
+Python and using the Django framework, Zulip supports both private
+messaging and group chats via conversation streams.
 
-[![CircleCI branch](https://img.shields.io/circleci/project/github/zulip/zulip/master.svg)](https://circleci.com/gh/zulip/zulip/tree/master)
-[![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip/master.svg)](https://codecov.io/gh/zulip/zulip/branch/master)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
+Zulip also supports fast search, drag-and-drop file uploads, image
+previews, group private messages, audible notifications,
+missed-message emails, desktop apps, and much more.
+
+Further information on the Zulip project and its features can be found
+at <https://www.zulip.org>.
+
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip)
+[![Coverage Status](https://img.shields.io/codecov/c/github/zulip/zulip.svg)](https://codecov.io/gh/zulip/zulip)
+[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)](http://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/)
+[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/)
 [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
+[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](http://twitter.com/zulip)
 
-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
+## Community
 
-## Getting started
+There are several places online where folks discuss Zulip.
 
-Click on the appropriate link below. If nothing seems to apply,
-join us on the
-[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
-and tell us what's up!
+* The primary place is the
+  [Zulip development community Zulip server][czo-doc] at
+  chat.zulip.org.
 
-You might be interested in:
+* For Google Summer of Code students and applicants, we have
+[a mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc)
+for help, questions, and announcements.  But it's often simpler to
+[visit chat.zulip.org][czo-doc] instead.
 
-* **Contributing code**. Check out our
-  [guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
-  to get started.  Zulip prides itself on maintaining a clean and
-  well-tested codebase, and a stock of hundreds of
-  [beginner-friendly issues][beginner-friendly].
+* We have a [public development discussion mailing list][zulip-devel],
+zulip-devel, which is currently pretty low traffic because most
+discussions happen in our public Zulip instance.  We use it to
+announce Zulip developer community gatherings and ask for feedback on
+major technical or design decisions.  It has several hundred
+subscribers, so you can use it to ask questions about features or
+possible bugs, but please don't use it ask for generic help getting
+started as a contributor (e.g. because you want to do Google Summer of
+Code).  The rest of this page covers how to get involved in the Zulip
+project in detail.
 
-* **Contributing non-code**.
-  [Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issues),
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
-  into your language,
-  [write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
-  for the Zulip blog, or
-  [give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
-  would love to hear from you, even if you're just trying the product out.
+* Zulip also has a [blog](https://blog.zulip.org/) and
+  [twitter account](https://twitter.com/zulip).
 
-* **Supporting Zulip**. Advocate for your organization to use Zulip, write a
-  review in the mobile app stores, or
-  [upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
-  product comparison sites.
+* Last but not least, we use [GitHub](https://github.com/zulip/zulip)
+to track Zulip-related issues (and store our code, of course).
+Anybody with a GitHub account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull Requests
+as our primary mechanism to receive code contributions.
 
-* **Checking Zulip out**. The best way to see Zulip in action is to drop by
-  the
-  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
-  also recommend reading Zulip for
-  [open source](https://zulip.com/for/open-source/), Zulip for
-  [companies](https://zulip.com/for/companies/), or Zulip for
-  [working groups and part time communities](https://zulip.com/for/working-groups-and-communities/).
+The Zulip community has a [Code of Conduct][code-of-conduct].
 
-* **Running a Zulip server**. Use a preconfigured [Digital Ocean droplet](https://marketplace.digitalocean.com/apps/zulip),
-  [install Zulip](https://zulip.readthedocs.io/en/stable/production/install.html)
-  directly, or use Zulip's
-  experimental [Docker image](https://zulip.readthedocs.io/en/latest/production/deployment.html#zulip-in-docker).
-  Commercial support is available; see <https://zulip.com/plans> for details.
+[zulip-devel]: https://groups.google.com/forum/#!forum/zulip-devel
 
-* **Using Zulip without setting up a server**. <https://zulip.com>
-  offers free and commercial hosting, including providing our paid
-  plan for free to fellow open source projects.
+## Installing the Zulip Development environment
 
-* **Participating in [outreach
-  programs](https://zulip.readthedocs.io/en/latest/overview/contributing.html#outreach-programs)**
-  like Google Summer of Code.
+The Zulip development environment is the recommended option for folks
+interested in trying out Zulip, since it is very easy to install.
+This is documented in [the developer installation guide][dev-install].
 
-You may also be interested in reading our [blog](https://blog.zulip.org/) or
-following us on [twitter](https://twitter.com/zulip).
-Zulip is distributed under the
-[Apache 2.0](https://github.com/zulip/zulip/blob/master/LICENSE) license.
+## Running Zulip in production
 
-[beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22
+Zulip in production supports Ubuntu 16.04 Xenial and Ubuntu 14.04
+Trusty. We're happy to support work to enable Zulip to run on
+additional platforms. The installation process is
+[documented here](https://zulip.readthedocs.io/en/latest/prod.html).
+
+## Ways to contribute
+
+Zulip welcomes all forms of contributions!  This page documents the
+Zulip development process.
+
+* **Pull requests**. Before a pull request can be merged, you need to
+sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
+We encourage early pull requests for work in progress. Prefix the title
+of your pull request with `[WIP]` and reference it when asking for
+community feedback. When you are ready for final review, remove
+the `[WIP]`.
+
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+docs][doc-test]. You can also try out [our new desktop
+client][electron], which is in alpha; we'd appreciate testing and
+[feedback](https://github.com/zulip/zulip-electron/issues/new).
+
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat]. You can also improve
+[Zulip.org][z-org].
+
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list](#community) and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list! Check out our [bug report guidelines][bug-report]
+before submitting. Please report any security issues you discover to
+zulip-security@googlegroups.com.
+
+* **App codebases**. This repository is for the Zulip server and web
+app (including most integrations).  The
+[beta React Native mobile app][mobile], [Java Android app][Android]
+(see [our mobile strategy][mobile-strategy]),
+[new Electron desktop app][electron], and
+[legacy Qt-based desktop app][desktop] are all separate repositories.
+
+* **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
+integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
+and [Trello][]), plus [node.js API bindings][node], an [isomorphic
+ JavaScript library][zulip-js], and a [full-text search PostgreSQL
+ extension][tsearch], as separate repos.
+
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation][transifex] if you're interested in
+contributing!
+
+* **Code Reviews**. Zulip is all about community and helping each
+other out.  Check out [#code review][code-review] on
+[chat.zulip.org][czo-doc] to help review PRs and give comments on
+other people's work. Everyone is welcome to participate, even those
+new to Zulip! Even just checking out the code, manually testing it,
+and posting on whether or not it worked is valuable.
+
+[cla]: https://opensource.dropbox.com/cla/
+[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
+[dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
+[electron]: https://github.com/zulip/zulip-electron
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[mobile]: https://github.com/zulip/zulip-mobile
+[mobile-strategy]: https://github.com/zulip/zulip-android/blob/master/android-strategy.md
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[hubot-adapter]: https://github.com/zulip/hubot-zulip
+[jenkins]: https://github.com/zulip/zulip-jenkins-plugin
+[node]: https://github.com/zulip/zulip-node
+[zulip-js]: https://github.com/zulip/zulip-js
+[phab]: https://github.com/zulip/phabricator-to-zulip
+[puppet]: https://github.com/matthewbarr/puppet-zulip
+[redmine]: https://github.com/zulip/zulip-redmine-plugin
+[trello]: https://github.com/zulip/trello-to-zulip
+[tsearch]: https://github.com/zulip/tsearch_extras
+[transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
+[z-org]: https://github.com/zulip/zulip.github.io
+[code-review]: https://chat.zulip.org/#narrow/stream/code.20review
+[bug-report]: http://zulip.readthedocs.io/en/latest/bug-reports.html
+
+## Google Summer of Code
+
+We participated in
+[GSoC](https://developers.google.com/open-source/gsoc/) in 2016 (with
+[great results](https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/))
+and [are participating](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
+in 2017 as well.
+
+## How to get involved with contributing to Zulip
+
+First, subscribe to the Zulip [development discussion mailing
+list](#community).
+
+The Zulip project uses a system of labels in our [issue
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
+
+* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Good first issue](https://github.com/zulip/zulip/labels/good%20first%20issue):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+  The Zulip project loves contributions of new documentation.
+
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
+  The projects that are
+  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
+  These are great projects if you're looking to make an impact.
+
+Another way to find issues in Zulip is to take advantage of our
+`area:<foo>` convention in separating out issues.  We partition all of
+our issues into areas like admin, compose, emoji, hotkeys, i18n,
+onboarding, search, etc.  Look through our
+[list of labels](https://github.com/zulip/zulip/labels), and click on
+some of the `area:` labels to see all the tickets related to your
+areas of interest.
+
+If you're excited about helping with an open issue, make sure to claim
+the issue by commenting the following in the comment section:
+"**@zulipbot** claim". **@zulipbot** will assign you to the issue and
+label the issue as **in progress**. For more details, check out
+[**@zulipbot**](https://github.com/zulip/zulipbot).
+
+You're encouraged to ask questions on how to best implement or debug
+your changes -- the Zulip maintainers are excited to answer questions
+to help you stay unblocked and working efficiently. It's great to ask
+questions in comments on GitHub issues and pull requests, or
+[on chat.zulip.org][czo-doc].  We'll direct longer discussions to
+Zulip chat, but please post a summary of what you learned from the
+chat, or link to the conversation, in a comment on the GitHub issue.
+
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
+
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
+
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to the
+[#design](https://chat.zulip.org/#narrow/stream/design) stream on
+[chat.zulip.org][czo-doc] to get broad feedback before you spend too
+much time on implementation details.
+
+Finally, before implementing a larger feature, we highly recommend
+looking at the
+[new feature tutorial](http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html)
+and [coding style guidelines](http://zulip.readthedocs.io/en/latest/code-style.html)
+on ReadTheDocs.
+
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just send an email
+to the [zulip-devel](#community) list with your thoughts.
+
+When you feel like you have completed your work on an issue, post your
+PR to the
+[#code review](https://chat.zulip.org/#narrow/stream/code.20review)
+stream on [chat.zulip.org][czo-doc].  This is our lightweight process
+that gives other developers the opportunity to give you comments and
+suggestions on your work.
+
+## License
+
+Copyright 2011-2017 Dropbox, Inc., Kandra Labs, Inc., and contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The software includes some works released by third parties under other
+free and open source licenses. Those works are redistributed under the
+license terms under which the works were received. For more details,
+see the ``docs/THIRDPARTY`` file included with this distribution.
+
+
+[czo-doc]: https://zulip.readthedocs.io/en/latest/chat-zulip-org.html

SECURITY.md

@@ -1,28 +0,0 @@
-# Security Policy
-
-Security announcements are sent to zulip-announce@googlegroups.com,
-so you should subscribe if you are running Zulip in production.
-
-## Reporting a Vulnerability
-
-We love responsible reports of (potential) security issues in Zulip,
-whether in the latest release or our development branch.
-
-Our security contact is security@zulip.com.  Reporters should expect a
-response within 24 hours.
-
-Please include details on the issue and how you'd like to be credited
-in our release notes when we publish the fix.
-
-Our [security
-model](https://zulip.readthedocs.io/en/latest/production/security-model.html)
-document may be a helpful resource.
-
-## Supported Versions
-
-Zulip provides security support for the latest major release, in the
-form of minor security/maintenance releases.
-
-We work hard to make
-[upgrades](https://zulip.readthedocs.io/en/latest/production/upgrade-or-modify.html#upgrading-to-a-release)
-reliable, so that there's no reason to run older major releases.

Vagrantfile

@@ -10,7 +10,7 @@ end
 if Vagrant::VERSION == "1.8.7" then
     path = `which curl`
     if path.include?('/opt/vagrant/embedded/bin/curl') then
-        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 "\
+        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
              "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
              "issue before provisioning. See "\
              "https://github.com/mitchellh/vagrant/issues/7997 "\
@@ -19,46 +19,16 @@ if Vagrant::VERSION == "1.8.7" then
     end
 end
 
-# Workaround: Vagrant removed the atlas.hashicorp.com to
-# vagrantcloud.com redirect in February 2018. The value of
-# DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
-# atlas.hashicorp.com, which means that removal broke the fetching and
-# updating of boxes (since the old URL doesn't work).  See
-# https://github.com/hashicorp/vagrant/issues/9442
-if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
-  Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
-end
-
-# Monkey patch https://github.com/hashicorp/vagrant/pull/10879 so we
-# can fall back to another provider if docker is not installed.
-begin
-  require Vagrant.source_root.join("plugins", "providers", "docker", "provider")
-rescue LoadError
-else
-  VagrantPlugins::DockerProvider::Provider.class_eval do
-    method(:usable?).owner == singleton_class or def self.usable?(raise_error=false)
-      VagrantPlugins::DockerProvider::Driver.new.execute("docker", "version")
-      true
-    rescue Vagrant::Errors::CommandUnavailable, VagrantPlugins::DockerProvider::Errors::ExecuteError
-      raise if raise_error
-      return false
-    end
-  end
-end
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
+  # For LXC. VirtualBox hosts use a different box, described below.
+  config.vm.box = "fgrehm/trusty64-lxc"
+
   # The Zulip development environment runs on 9991 on the guest.
   host_port = 9991
   http_proxy = https_proxy = no_proxy = nil
   host_ip_addr = "127.0.0.1"
 
-  # System settings for the virtual machine.
-  vm_num_cpus = "2"
-  vm_memory = "2048"
-
-  ubuntu_mirror = ""
-
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/srv/zulip"
 
@@ -74,9 +44,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       when "NO_PROXY"; no_proxy = value
       when "HOST_PORT"; host_port = value.to_i
       when "HOST_IP_ADDR"; host_ip_addr = value
-      when "GUEST_CPUS"; vm_num_cpus = value
-      when "GUEST_MEMORY_MB"; vm_memory = value
-      when "UBUNTU_MIRROR"; ubuntu_mirror = value
       end
     end
   end
@@ -101,48 +68,42 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
-  # Specify Docker provider before VirtualBox provider so it's preferred.
-  config.vm.provider "docker" do |d, override|
-    d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
-    d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
-    if !ubuntu_mirror.empty?
-      d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
+  # Specify LXC provider before VirtualBox provider so it's preferred.
+  config.vm.provider "lxc" do |lxc|
+    if command? "lxc-ls"
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
+      if LXC_VERSION >= "1.1.0"
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
+        lxc.customize 'aa_allow_incomplete', 1
+      end
+      if LXC_VERSION >= "2.0.0"
+        lxc.backingstore = 'dir'
+      end
     end
-    d.has_ssh = true
-    d.create_args = ["--ulimit", "nofile=1024:65536"]
   end
 
   config.vm.provider "virtualbox" do |vb, override|
-    override.vm.box = "hashicorp/bionic64"
+    override.vm.box = "ubuntu/trusty64"
     # It's possible we can get away with just 1.5GB; more testing needed
-    vb.memory = vm_memory
-    vb.cpus = vm_num_cpus
+    vb.memory = 2048
+    vb.cpus = 2
+  end
+
+  config.vm.provider "vmware_fusion" do |vb, override|
+    override.vm.box = "puphpet/ubuntu1404-x64"
+    vb.vmx["memsize"] = "2048"
+    vb.vmx["numvcpus"] = "2"
   end
 
 $provision_script = <<SCRIPT
 set -x
 set -e
 set -o pipefail
-
-# Code should go here, rather than tools/provision, only if it is
-# something that we don't want to happen when running provision in a
-# development environment not using Vagrant.
-
-# Set the Ubuntu mirror
-[ ! '#{ubuntu_mirror}' ] || sudo sed -i 's|http://\\(\\w*\\.\\)*archive\\.ubuntu\\.com/ubuntu/\\? |#{ubuntu_mirror} |' /etc/apt/sources.list
-
-# Set the MOTD on the system to have Zulip instructions
-sudo ln -nsf /srv/zulip/tools/setup/dev-motd /etc/update-motd.d/99-zulip-dev
-sudo rm -f /etc/update-motd.d/10-help-text
-sudo dpkg --purge landscape-client landscape-common ubuntu-release-upgrader-core update-manager-core update-notifier-common ubuntu-server
-sudo dpkg-divert --add --rename /etc/default/motd-news
-sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
-
 # If the host is running SELinux remount the /sys/fs/selinux directory as read only,
 # needed for apt-get to work.
 if [ -d "/sys/fs/selinux" ]; then
-    sudo mount -o remount,ro /sys/fs/selinux
+  sudo mount -o remount,ro /sys/fs/selinux
 fi
 
 # Set default locale, this prevents errors if the user has another locale set.
@@ -150,35 +111,14 @@ if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
     echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
 fi
 
-# Set an environment variable, so that we won't print the virtualenv
-# shell warning (it'll be wrong, since the shell is dying anyway)
-export SKIP_VENV_SHELL_WARNING=1
-
-# End `set -x`, so that the end of provision doesn't look like an error
-# message after a successful run.
-set +x
-
-# Check if the zulip directory is writable
-if [ ! -w /srv/zulip ]; then
-    echo "The vagrant user is unable to write to the zulip directory."
-    echo "To fix this, run the following commands on the host machine:"
-    # sudo is required since our uid is not 1000
-    echo '    vagrant halt -f'
-    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
-    echo '    sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
-    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
-    echo "You can resume setting up your vagrant environment by running:"
-    echo "    vagrant up"
-    exit 1
-fi
 # Provision the development environment
 ln -nsf /srv/zulip ~/zulip
 /srv/zulip/tools/provision
 
 # Run any custom provision hooks the user has configured
 if [ -f /srv/zulip/tools/custom_provision ]; then
-    chmod +x /srv/zulip/tools/custom_provision
-    /srv/zulip/tools/custom_provision
+  chmod +x /srv/zulip/tools/custom_provision
+  /srv/zulip/tools/custom_provision
 fi
 SCRIPT
 

analytics/lib/counts.py

@@ -1,58 +1,43 @@
-import logging
-import time
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
-from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union
-
 from django.conf import settings
-from django.db import connection
+from django.db import connection, models
 from django.db.models import F
-from psycopg2.sql import SQL, Composable, Identifier, Literal
 
-from analytics.models import (
-    BaseCount,
-    FillState,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-    installation_epoch,
-    last_successful_fill,
-)
-from zerver.lib.logging_util import log_to_file
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
-from zerver.models import (
-    Message,
-    Realm,
-    RealmAuditLog,
-    Stream,
-    UserActivityInterval,
-    UserProfile,
-    models,
-)
+from analytics.models import InstallationCount, RealmCount, \
+    UserCount, StreamCount, BaseCount, FillState, Anomaly, installation_epoch, \
+    last_successful_fill
+from zerver.models import Realm, UserProfile, Message, Stream, \
+    UserActivityInterval, RealmAuditLog, models
+from zerver.lib.timestamp import floor_to_day, floor_to_hour, ceiling_to_day, \
+    ceiling_to_hour, verify_UTC
+
+from typing import Any, Callable, Dict, List, Optional, Text, Tuple, Type, Union
+
+from collections import defaultdict, OrderedDict
+from datetime import timedelta, datetime
+from zerver.lib.logging_util import create_logger
+import time
 
 ## Logging setup ##
 
-logger = logging.getLogger('zulip.management')
-log_to_file(logger, settings.ANALYTICS_LOG_PATH)
+logger = create_logger('zulip.management', settings.ANALYTICS_LOG_PATH, 'INFO')
 
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365*1000)
 
 ## Class definitions ##
 
-class CountStat:
+class CountStat(object):
     HOUR = 'hour'
     DAY = 'day'
     FREQUENCIES = frozenset([HOUR, DAY])
 
-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta]=None) -> None:
+    def __init__(self, property, data_collector, frequency, interval=None):
+        # type: (str, DataCollector, str, Optional[timedelta]) -> None
         self.property = property
         self.data_collector = data_collector
         # might have to do something different for bitfields
         if frequency not in self.FREQUENCIES:
-            raise AssertionError(f"Unknown frequency: {frequency}")
+            raise AssertionError("Unknown frequency: %s" % (frequency,))
         self.frequency = frequency
         if interval is not None:
             self.interval = interval
@@ -61,48 +46,41 @@ class CountStat:
         else:  # frequency == CountStat.DAY
             self.interval = timedelta(days=1)
 
-    def __str__(self) -> str:
-        return f"<CountStat: {self.property}>"
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<CountStat: %s>" % (self.property,)
 
 class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
+    def __init__(self, property, output_table, frequency):
+        # type: (str, Type[BaseCount], str) -> None
         CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 
 class DependentCountStat(CountStat):
-    def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
-                 interval: Optional[timedelta] = None, dependencies: Sequence[str] = []) -> None:
+    def __init__(self, property, data_collector, frequency, interval=None, dependencies=[]):
+        # type: (str, DataCollector, str, Optional[timedelta], List[str]) -> None
         CountStat.__init__(self, property, data_collector, frequency, interval=interval)
         self.dependencies = dependencies
 
-class DataCollector:
-    def __init__(self, output_table: Type[BaseCount],
-                 pull_function: Optional[Callable[[str, datetime, datetime, Optional[Realm]], int]]) -> None:
+class DataCollector(object):
+    def __init__(self, output_table, pull_function):
+        # type: (Type[BaseCount], Optional[Callable[[str, datetime, datetime], int]]) -> None
         self.output_table = output_table
         self.pull_function = pull_function
 
 ## CountStat-level operations ##
 
-def process_count_stat(stat: CountStat, fill_to_time: datetime,
-                       realm: Optional[Realm]=None) -> None:
-    # TODO: The realm argument is not yet supported, in that we don't
-    # have a solution for how to update FillState if it is passed.  It
-    # exists solely as partial plumbing for when we do fully implement
-    # doing single-realm analytics runs for use cases like data import.
-    #
-    # Also, note that for the realm argument to be properly supported,
-    # the CountStat object passed in needs to have come from
-    # E.g. get_count_stats(realm), i.e. have the realm_id already
-    # entered into the SQL query defined by the CountState object.
+def process_count_stat(stat, fill_to_time):
+    # type: (CountStat, datetime) -> None
     if stat.frequency == CountStat.HOUR:
         time_increment = timedelta(hours=1)
     elif stat.frequency == CountStat.DAY:
         time_increment = timedelta(days=1)
     else:
-        raise AssertionError(f"Unknown frequency: {stat.frequency}")
+        raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
 
     verify_UTC(fill_to_time)
     if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError(f"fill_to_time must be on an hour boundary: {fill_to_time}")
+        raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
 
     fill_state = FillState.objects.filter(property=stat.property).first()
     if fill_state is None:
@@ -110,56 +88,59 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime,
         fill_state = FillState.objects.create(property=stat.property,
                                               end_time=currently_filled,
                                               state=FillState.DONE)
-        logger.info("INITIALIZED %s %s", stat.property, currently_filled)
+        logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
     elif fill_state.state == FillState.STARTED:
-        logger.info("UNDO START %s %s", stat.property, fill_state.end_time)
+        logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
         do_delete_counts_at_hour(stat, fill_state.end_time)
         currently_filled = fill_state.end_time - time_increment
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        logger.info("UNDO DONE %s", stat.property)
+        logger.info("UNDO DONE %s" % (stat.property,))
     elif fill_state.state == FillState.DONE:
         currently_filled = fill_state.end_time
     else:
-        raise AssertionError(f"Unknown value for FillState.state: {fill_state.state}.")
+        raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
 
     if isinstance(stat, DependentCountStat):
         for dependency in stat.dependencies:
             dependency_fill_time = last_successful_fill(dependency)
             if dependency_fill_time is None:
-                logger.warning("DependentCountStat %s run before dependency %s.",
-                               stat.property, dependency)
+                logger.warning("DependentCountStat %s run before dependency %s." %
+                               (stat.property, dependency))
                 return
             fill_to_time = min(fill_to_time, dependency_fill_time)
 
     currently_filled = currently_filled + time_increment
     while currently_filled <= fill_to_time:
-        logger.info("START %s %s", stat.property, currently_filled)
+        logger.info("START %s %s" % (stat.property, currently_filled))
         start = time.time()
         do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
-        do_fill_count_stat_at_hour(stat, currently_filled, realm)
+        do_fill_count_stat_at_hour(stat, currently_filled)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
         currently_filled = currently_filled + time_increment
-        logger.info("DONE %s (%dms)", stat.property, (end-start)*1000)
+        logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
 
-def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
+def do_update_fill_state(fill_state, end_time, state):
+    # type: (FillState, datetime, int) -> None
     fill_state.end_time = end_time
     fill_state.state = state
     fill_state.save()
 
 # We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
 # and is timezone aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(stat: CountStat, end_time: datetime, realm: Optional[Realm]=None) -> None:
+def do_fill_count_stat_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
     start_time = end_time - stat.interval
     if not isinstance(stat, LoggingCountStat):
         timer = time.time()
         assert(stat.data_collector.pull_function is not None)
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time, realm)
-        logger.info("%s run pull_function (%dms/%sr)",
-                    stat.property, (time.time()-timer)*1000, rows_added)
-    do_aggregate_to_summary_table(stat, end_time, realm)
+        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
+        logger.info("%s run pull_function (%dms/%sr)" %
+                    (stat.property, (time.time()-timer)*1000, rows_added))
+    do_aggregate_to_summary_table(stat, end_time)
 
-def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
+def do_delete_counts_at_hour(stat, end_time):
+    # type: (CountStat, datetime) -> None
     if isinstance(stat, LoggingCountStat):
         InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
         if stat.data_collector.output_table in [UserCount, StreamCount]:
@@ -170,87 +151,57 @@ def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
         RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
         InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
 
-def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime,
-                                  realm: Optional[Realm]=None) -> None:
+def do_aggregate_to_summary_table(stat, end_time):
+    # type: (CountStat, datetime) -> None
     cursor = connection.cursor()
 
     # Aggregate into RealmCount
     output_table = stat.data_collector.output_table
-    if realm is not None:
-        realm_clause = SQL("AND zerver_realm.id = {}").format(Literal(realm.id))
-    else:
-        realm_clause = SQL("")
-
     if output_table in (UserCount, StreamCount):
-        realmcount_query = SQL("""
+        realmcount_query = """
             INSERT INTO analytics_realmcount
                 (realm_id, value, property, subgroup, end_time)
             SELECT
-                zerver_realm.id, COALESCE(sum({output_table}.value), 0), %(property)s,
-                {output_table}.subgroup, %(end_time)s
+                zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
+                %(output_table)s.subgroup, %%(end_time)s
             FROM zerver_realm
-            JOIN {output_table}
+            JOIN %(output_table)s
             ON
-                zerver_realm.id = {output_table}.realm_id
+                zerver_realm.id = %(output_table)s.realm_id
             WHERE
-                {output_table}.property = %(property)s AND
-                {output_table}.end_time = %(end_time)s
-                {realm_clause}
-            GROUP BY zerver_realm.id, {output_table}.subgroup
-        """).format(
-            output_table=Identifier(output_table._meta.db_table),
-            realm_clause=realm_clause,
-        )
+                %(output_table)s.property = '%(property)s' AND
+                %(output_table)s.end_time = %%(end_time)s
+            GROUP BY zerver_realm.id, %(output_table)s.subgroup
+        """ % {'output_table': output_table._meta.db_table,
+               'property': stat.property}
         start = time.time()
-        cursor.execute(realmcount_query, {
-            'property': stat.property,
-            'end_time': end_time,
-        })
+        cursor.execute(realmcount_query, {'end_time': end_time})
         end = time.time()
-        logger.info(
-            "%s RealmCount aggregation (%dms/%sr)",
-            stat.property, (end - start) * 1000, cursor.rowcount,
-        )
-
-    if realm is None:
-        # Aggregate into InstallationCount.  Only run if we just
-        # processed counts for all realms.
-        #
-        # TODO: Add support for updating installation data after
-        # changing an individual realm's values.
-        installationcount_query = SQL("""
-            INSERT INTO analytics_installationcount
-                (value, property, subgroup, end_time)
-            SELECT
-                sum(value), %(property)s, analytics_realmcount.subgroup, %(end_time)s
-            FROM analytics_realmcount
-            WHERE
-                property = %(property)s AND
-                end_time = %(end_time)s
-            GROUP BY analytics_realmcount.subgroup
-        """)
-        start = time.time()
-        cursor.execute(installationcount_query, {
-            'property': stat.property,
-            'end_time': end_time,
-        })
-        end = time.time()
-        logger.info(
-            "%s InstallationCount aggregation (%dms/%sr)",
-            stat.property, (end - start) * 1000, cursor.rowcount,
-        )
+        logger.info("%s RealmCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
 
+    # Aggregate into InstallationCount
+    installationcount_query = """
+        INSERT INTO analytics_installationcount
+            (value, property, subgroup, end_time)
+        SELECT
+            sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
+        FROM analytics_realmcount
+        WHERE
+            property = '%(property)s' AND
+            end_time = %%(end_time)s
+        GROUP BY analytics_realmcount.subgroup
+    """ % {'property': stat.property}
+    start = time.time()
+    cursor.execute(installationcount_query, {'end_time': end_time})
+    end = time.time()
+    logger.info("%s InstallationCount aggregation (%dms/%sr)" % (stat.property, (end-start)*1000, cursor.rowcount))
     cursor.close()
 
 ## Utility functions called from outside counts.py ##
 
 # called from zerver/lib/actions.py; should not throw any errors
-def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream], stat: CountStat,
-                              subgroup: Optional[Union[str, int, bool]], event_time: datetime,
-                              increment: int=1) -> None:
-    if not increment:
-        return
-
+def do_increment_logging_stat(zerver_object, stat, subgroup, event_time, increment=1):
+    # type: (Union[Realm, UserProfile, Stream], CountStat, Optional[Union[str, int, bool]], datetime, int) -> None
     table = stat.data_collector.output_table
     if table == RealmCount:
         id_args = {'realm': zerver_object}
@@ -271,14 +222,17 @@ def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream],
         row.value = F('value') + increment
         row.save(update_fields=['value'])
 
-def do_drop_all_analytics_tables() -> None:
+def do_drop_all_analytics_tables():
+    # type: () -> None
     UserCount.objects.all().delete()
     StreamCount.objects.all().delete()
     RealmCount.objects.all().delete()
     InstallationCount.objects.all().delete()
     FillState.objects.all().delete()
+    Anomaly.objects.all().delete()
 
-def do_drop_single_stat(property: str) -> None:
+def do_drop_single_stat(property):
+    # type: (str) -> None
     UserCount.objects.filter(property=property).delete()
     StreamCount.objects.filter(property=property).delete()
     RealmCount.objects.filter(property=property).delete()
@@ -287,71 +241,48 @@ def do_drop_single_stat(property: str) -> None:
 
 ## DataCollector-level operations ##
 
-QueryFn = Callable[[Dict[str, Composable]], Composable]
-
-def do_pull_by_sql_query(
-    property: str,
-    start_time: datetime,
-    end_time: datetime,
-    query: QueryFn,
-    group_by: Optional[Tuple[models.Model, str]],
-) -> int:
+def do_pull_by_sql_query(property, start_time, end_time, query, group_by):
+    # type: (str, datetime, datetime, str, Optional[Tuple[models.Model, str]]) -> int
     if group_by is None:
-        subgroup = SQL('NULL')
-        group_by_clause = SQL('')
+        subgroup = 'NULL'
+        group_by_clause  = ''
     else:
-        subgroup = Identifier(group_by[0]._meta.db_table, group_by[1])
-        group_by_clause = SQL(', {}').format(subgroup)
+        subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
+        group_by_clause = ', ' + subgroup
 
     # We do string replacement here because cursor.execute will reject a
     # group_by_clause given as a param.
     # We pass in the datetimes as params to cursor.execute so that we don't have to
     # think about how to convert python datetimes to SQL datetimes.
-    query_ = query({
-        'subgroup': subgroup,
-        'group_by_clause': group_by_clause,
-    })
+    query_ = query % {'property': property, 'subgroup': subgroup,
+                      'group_by_clause': group_by_clause}
     cursor = connection.cursor()
-    cursor.execute(query_, {
-        'property': property,
-        'time_start': start_time,
-        'time_end': end_time,
-    })
+    cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
     rowcount = cursor.rowcount
     cursor.close()
     return rowcount
 
-def sql_data_collector(
-    output_table: Type[BaseCount],
-    query: QueryFn,
-    group_by: Optional[Tuple[models.Model, str]],
-) -> DataCollector:
-    def pull_function(property: str, start_time: datetime, end_time: datetime,
-                      realm: Optional[Realm] = None) -> int:
-        # The pull function type needs to accept a Realm argument
-        # because the 'minutes_active::day' CountStat uses
-        # DataCollector directly for do_pull_minutes_active, which
-        # requires the realm argument.  We ignore it here, because the
-        # realm should have been already encoded in the `query` we're
-        # passed.
+def sql_data_collector(output_table, query, group_by):
+    # type: (Type[BaseCount], str, Optional[Tuple[models.Model, str]]) -> DataCollector
+    def pull_function(property, start_time, end_time):
+        # type: (str, datetime, datetime) -> int
         return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
     return DataCollector(output_table, pull_function)
 
-def do_pull_minutes_active(property: str, start_time: datetime, end_time: datetime,
-                           realm: Optional[Realm] = None) -> int:
+def do_pull_minutes_active(property, start_time, end_time):
+    # type: (str, datetime, datetime) -> int
     user_activity_intervals = UserActivityInterval.objects.filter(
-        end__gt=start_time, start__lt=end_time,
+        end__gt=start_time, start__lt=end_time
     ).select_related(
-        'user_profile',
+        'user_profile'
     ).values_list(
         'user_profile_id', 'user_profile__realm_id', 'start', 'end')
 
-    seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
+    seconds_active = defaultdict(float)  # type: Dict[Tuple[int, int], float]
     for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        if realm is None or realm.id == realm_id:
-            start = max(start_time, interval_start)
-            end = min(end_time, interval_end)
-            seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
+        start = max(start_time, interval_start)
+        end = min(end_time, interval_end)
+        seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
 
     rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
                       end_time=end_time, value=int(seconds // 60))
@@ -359,39 +290,27 @@ def do_pull_minutes_active(property: str, start_time: datetime, end_time: dateti
     UserCount.objects.bulk_create(rows)
     return len(rows)
 
-def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+count_message_by_user_query = """
     INSERT INTO analytics_usercount
         (user_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
-        %(property)s, {subgroup}, %(time_end)s
+        zerver_userprofile.id, zerver_userprofile.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_userprofile
     JOIN zerver_message
     ON
         zerver_userprofile.id = zerver_message.sender_id
     WHERE
-        zerver_userprofile.date_joined < %(time_end)s AND
-        zerver_message.date_sent >= %(time_start)s AND
-        {realm_clause}
-        zerver_message.date_sent < %(time_end)s
-    GROUP BY zerver_userprofile.id {group_by_clause}
-""").format(**kwargs, realm_clause=realm_clause)
+        zerver_userprofile.date_joined < %%(time_end)s AND
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s
+    GROUP BY zerver_userprofile.id %(group_by_clause)s
+"""
 
 # Note: ignores the group_by / group_by_clause.
-def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+count_message_type_by_user_query = """
     INSERT INTO analytics_usercount
             (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, %(property)s, message_type, %(time_end)s
+    SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
     FROM
     (
         SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
@@ -409,36 +328,28 @@ def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
         JOIN zerver_message
         ON
             zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.date_sent >= %(time_start)s AND
-            {realm_clause}
-            zerver_message.date_sent < %(time_end)s
+            zerver_message.pub_date >= %%(time_start)s AND
+            zerver_message.pub_date < %%(time_end)s
         JOIN zerver_recipient
         ON
             zerver_message.recipient_id = zerver_recipient.id
         LEFT JOIN zerver_stream
         ON
             zerver_recipient.type_id = zerver_stream.id
-        GROUP BY
-            zerver_userprofile.realm_id, zerver_userprofile.id,
-            zerver_recipient.type, zerver_stream.invite_only
+        GROUP BY zerver_userprofile.realm_id, zerver_userprofile.id, zerver_recipient.type, zerver_stream.invite_only
     ) AS subquery
     GROUP BY realm_id, id, message_type
-""").format(**kwargs, realm_clause=realm_clause)
+"""
 
 # This query joins to the UserProfile table since all current queries that
 # use this also subgroup on UserProfile.is_bot. If in the future there is a
 # stat that counts messages by stream and doesn't need the UserProfile
 # table, consider writing a new query for efficiency.
-def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+count_message_by_stream_query = """
     INSERT INTO analytics_streamcount
         (stream_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), %(property)s, {subgroup}, %(time_end)s
+        zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_stream
     JOIN zerver_recipient
     ON
@@ -450,61 +361,48 @@ def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
     ON
         zerver_message.sender_id = zerver_userprofile.id
     WHERE
-        zerver_stream.date_created < %(time_end)s AND
+        zerver_stream.date_created < %%(time_end)s AND
         zerver_recipient.type = 2 AND
-        zerver_message.date_sent >= %(time_start)s AND
-        {realm_clause}
-        zerver_message.date_sent < %(time_end)s
-    GROUP BY zerver_stream.id {group_by_clause}
-""").format(**kwargs, realm_clause=realm_clause)
+        zerver_message.pub_date >= %%(time_start)s AND
+        zerver_message.pub_date < %%(time_end)s
+    GROUP BY zerver_stream.id %(group_by_clause)s
+"""
 
 # Hardcodes the query needed by active_users:is_bot:day, since that is
 # currently the only stat that uses this.
-def count_user_by_realm_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+count_user_by_realm_query = """
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
+        zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_realm
     JOIN zerver_userprofile
     ON
         zerver_realm.id = zerver_userprofile.realm_id
     WHERE
-        zerver_realm.date_created < %(time_end)s AND
-        zerver_userprofile.date_joined >= %(time_start)s AND
-        zerver_userprofile.date_joined < %(time_end)s AND
-        {realm_clause}
+        zerver_realm.date_created < %%(time_end)s AND
+        zerver_userprofile.date_joined >= %%(time_start)s AND
+        zerver_userprofile.date_joined < %%(time_end)s AND
         zerver_userprofile.is_active = TRUE
-    GROUP BY zerver_realm.id {group_by_clause}
-""").format(**kwargs, realm_clause=realm_clause)
+    GROUP BY zerver_realm.id %(group_by_clause)s
+"""
 
 # Currently hardcodes the query needed for active_users_audit:is_bot:day.
 # Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
+# event_type in ['user_created', 'user_deactivated', etc].
 # In particular, it's important to ensure that migrations don't cause that to happen.
-def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+check_realmauditlog_by_user_query = """
     INSERT INTO analytics_usercount
         (user_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
+        ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_realmauditlog ral1
     JOIN (
         SELECT modified_user_id, max(event_time) AS max_event_time
         FROM zerver_realmauditlog
         WHERE
-            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
-            {realm_clause}
-            event_time < %(time_end)s
+            event_type in ('user_created', 'user_deactivated', 'user_activated', 'user_reactivated') AND
+            event_time < %%(time_end)s
         GROUP BY modified_user_id
     ) ral2
     ON
@@ -514,181 +412,123 @@ def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
     ON
         ral1.modified_user_id = zerver_userprofile.id
     WHERE
-        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
-    """).format(
-        **kwargs,
-        user_created=Literal(RealmAuditLog.USER_CREATED),
-        user_activated=Literal(RealmAuditLog.USER_ACTIVATED),
-        user_deactivated=Literal(RealmAuditLog.USER_DEACTIVATED),
-        user_reactivated=Literal(RealmAuditLog.USER_REACTIVATED),
-        realm_clause=realm_clause,
-    )
+        ral1.event_type in ('user_created', 'user_activated', 'user_reactivated')
+"""
 
-def check_useractivityinterval_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+check_useractivityinterval_by_user_query = """
     INSERT INTO analytics_usercount
         (user_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
+        zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_userprofile
     JOIN zerver_useractivityinterval
     ON
         zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
     WHERE
-        zerver_useractivityinterval.end >= %(time_start)s AND
-        {realm_clause}
-        zerver_useractivityinterval.start < %(time_end)s
-    GROUP BY zerver_userprofile.id {group_by_clause}
-""").format(**kwargs, realm_clause=realm_clause)
+        zerver_useractivityinterval.end >= %%(time_start)s AND
+        zerver_useractivityinterval.start < %%(time_end)s
+    GROUP BY zerver_userprofile.id %(group_by_clause)s
+"""
 
-def count_realm_active_humans_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause = SQL("")
-    else:
-        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL("""
+count_realm_active_humans_query = """
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-        usercount1.realm_id, count(*), %(property)s, NULL, %(time_end)s
+        usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
     FROM (
         SELECT realm_id, user_id
         FROM analytics_usercount
         WHERE
             property = 'active_users_audit:is_bot:day' AND
             subgroup = 'false' AND
-            {realm_clause}
-            end_time = %(time_end)s
+            end_time = %%(time_end)s
     ) usercount1
     JOIN (
         SELECT realm_id, user_id
         FROM analytics_usercount
         WHERE
             property = '15day_actives::day' AND
-            {realm_clause}
-            end_time = %(time_end)s
+            end_time = %%(time_end)s
     ) usercount2
     ON
         usercount1.user_id = usercount2.user_id
     GROUP BY usercount1.realm_id
-""").format(**kwargs, realm_clause=realm_clause)
+"""
 
 # Currently unused and untested
-count_stream_by_realm_query = lambda kwargs: SQL("""
+count_stream_by_realm_query = """
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
+        zerver_realm.id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
     FROM zerver_realm
     JOIN zerver_stream
     ON
         zerver_realm.id = zerver_stream.realm_id AND
     WHERE
-        zerver_realm.date_created < %(time_end)s AND
-        zerver_stream.date_created >= %(time_start)s AND
-        zerver_stream.date_created < %(time_end)s
-    GROUP BY zerver_realm.id {group_by_clause}
-""").format(**kwargs)
+        zerver_realm.date_created < %%(time_end)s AND
+        zerver_stream.date_created >= %%(time_start)s AND
+        zerver_stream.date_created < %%(time_end)s
+    GROUP BY zerver_realm.id %(group_by_clause)s
+"""
 
-def get_count_stats(realm: Optional[Realm]=None) -> Dict[str, CountStat]:
-    ## CountStat declarations ##
+## CountStat declarations ##
 
-    count_stats_ = [
-        # Messages Sent stats
-        # Stats that count the number of messages sent in various ways.
-        # These are also the set of stats that read from the Message table.
+count_stats_ = [
+    # Messages Sent stats
+    # Stats that count the number of messages sent in various ways.
+    # These are also the set of stats that read from the Message table.
 
-        CountStat('messages_sent:is_bot:hour',
-                  sql_data_collector(UserCount, count_message_by_user_query(
-                      realm), (UserProfile, 'is_bot')),
-                  CountStat.HOUR),
-        CountStat('messages_sent:message_type:day',
-                  sql_data_collector(
-                      UserCount, count_message_type_by_user_query(realm), None),
-                  CountStat.DAY),
-        CountStat('messages_sent:client:day',
-                  sql_data_collector(UserCount, count_message_by_user_query(realm),
-                                     (Message, 'sending_client_id')), CountStat.DAY),
-        CountStat('messages_in_stream:is_bot:day',
-                  sql_data_collector(StreamCount, count_message_by_stream_query(realm),
-                                     (UserProfile, 'is_bot')), CountStat.DAY),
+    CountStat('messages_sent:is_bot:hour',
+              sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
+              CountStat.HOUR),
+    CountStat('messages_sent:message_type:day',
+              sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
+    CountStat('messages_sent:client:day',
+              sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
+              CountStat.DAY),
+    CountStat('messages_in_stream:is_bot:day',
+              sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
+              CountStat.DAY),
 
-        # Number of Users stats
-        # Stats that count the number of active users in the UserProfile.is_active sense.
+    # Number of Users stats
+    # Stats that count the number of active users in the UserProfile.is_active sense.
 
-        # 'active_users_audit:is_bot:day' is the canonical record of which users were
-        # active on which days (in the UserProfile.is_active sense).
-        # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-        CountStat('active_users_audit:is_bot:day',
-                  sql_data_collector(UserCount, check_realmauditlog_by_user_query(
-                      realm), (UserProfile, 'is_bot')),
-                  CountStat.DAY),
+    # 'active_users_audit:is_bot:day' is the canonical record of which users were
+    # active on which days (in the UserProfile.is_active sense).
+    # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
+    CountStat('active_users_audit:is_bot:day',
+              sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
+              CountStat.DAY),
+    # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
+    # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
+    # sum sequence of 'active_users_log:is_bot:day', for any realm that
+    # started after the latter stat was introduced.
+    LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
+    # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
+    # approximation, e.g. if a user is deactivated between the end of the
+    # day and when this stat is run, they won't be counted. However, is the
+    # simplest of the three to inspect by hand.
+    CountStat('active_users:is_bot:day',
+              sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
+              CountStat.DAY, interval=TIMEDELTA_MAX),
 
-        # Important note: LoggingCountStat objects aren't passed the
-        # Realm argument, because by nature they have a logging
-        # structure, not a pull-from-database structure, so there's no
-        # way to compute them for a single realm after the fact (the
-        # use case for passing a Realm argument).
+    # User Activity stats
+    # Stats that measure user activity in the UserActivityInterval sense.
 
-        # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-        # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-        # sum sequence of 'active_users_log:is_bot:day', for any realm that
-        # started after the latter stat was introduced.
-        LoggingCountStat('active_users_log:is_bot:day',
-                         RealmCount, CountStat.DAY),
-        # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-        # approximation, e.g. if a user is deactivated between the end of the
-        # day and when this stat is run, they won't be counted. However, is the
-        # simplest of the three to inspect by hand.
-        CountStat('active_users:is_bot:day',
-                  sql_data_collector(RealmCount, count_user_by_realm_query(realm), (UserProfile, 'is_bot')),
-                  CountStat.DAY, interval=TIMEDELTA_MAX),
+    CountStat('15day_actives::day',
+              sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
+              CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
+    CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
 
-        # Messages read stats.  messages_read::hour is the total
-        # number of messages read, whereas
-        # messages_read_interactions::hour tries to count the total
-        # number of UI interactions resulting in messages being marked
-        # as read (imperfect because of batching of some request
-        # types, but less likely to be overwhelmed by a single bulk
-        # operation).
-        LoggingCountStat('messages_read::hour', UserCount, CountStat.HOUR),
-        LoggingCountStat('messages_read_interactions::hour', UserCount, CountStat.HOUR),
+    # Dependent stats
+    # Must come after their dependencies.
 
-        # User Activity stats
-        # Stats that measure user activity in the UserActivityInterval sense.
+    # Canonical account of the number of active humans in a realm on each day.
+    DependentCountStat('realm_active_humans::day',
+                       sql_data_collector(RealmCount, count_realm_active_humans_query, None),
+                       CountStat.DAY,
+                       dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
+]
 
-        CountStat('1day_actives::day',
-                  sql_data_collector(
-                      UserCount, check_useractivityinterval_by_user_query(realm), None),
-                  CountStat.DAY, interval=timedelta(days=1)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-        CountStat('15day_actives::day',
-                  sql_data_collector(
-                      UserCount, check_useractivityinterval_by_user_query(realm), None),
-                  CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
-        CountStat('minutes_active::day', DataCollector(
-            UserCount, do_pull_minutes_active), CountStat.DAY),
-
-        # Rate limiting stats
-
-        # Used to limit the number of invitation emails sent by a realm
-        LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
-
-        # Dependent stats
-        # Must come after their dependencies.
-
-        # Canonical account of the number of active humans in a realm on each day.
-        DependentCountStat('realm_active_humans::day',
-                           sql_data_collector(
-                               RealmCount, count_realm_active_humans_query(realm), None),
-                           CountStat.DAY,
-                           dependencies=['active_users_audit:is_bot:day', '15day_actives::day']),
-    ]
-
-    return OrderedDict([(stat.property, stat) for stat in count_stats_])
-
-# To avoid refactoring for now COUNT_STATS can be used as before
-COUNT_STATS = get_count_stats()
+COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])

analytics/lib/fixtures.py

@@ -1,15 +1,19 @@
+from zerver.models import Realm, UserProfile, Stream, Message
+from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+
+from datetime import datetime
 from math import sqrt
 from random import gauss, random, seed
 from typing import List
 
-from analytics.lib.counts import CountStat
+from six.moves import zip
 
-
-def generate_time_series_data(days: int=100, business_hours_base: float=10,
-                              non_business_hours_base: float=10, growth: float=1,
-                              autocorrelation: float=0, spikiness: float=1,
-                              holiday_rate: float=0, frequency: str=CountStat.DAY,
-                              partial_sum: bool=False, random_seed: int=26) -> List[int]:
+def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
+                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
+                              frequency=CountStat.DAY, partial_sum=False, random_seed=26):
+    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
 
@@ -44,10 +48,10 @@ def generate_time_series_data(days: int=100, business_hours_base: float=10,
                       [24*non_business_hours_base] * 2
         holidays = [random() < holiday_rate for i in range(days)]
     else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
+        raise AssertionError("Unknown frequency: %s" % (frequency,))
     if length < 2:
         raise AssertionError("Must be generating at least 2 data points. "
-                             f"Currently generating {length}")
+                             "Currently generating %s" % (length,))
     growth_base = growth ** (1. / (length-1))
     values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
 

analytics/lib/time_utils.py

@@ -1,16 +1,16 @@
+from zerver.lib.timestamp import floor_to_hour, floor_to_day, \
+    timestamp_to_datetime, verify_UTC
+from analytics.lib.counts import CountStat
+
 from datetime import datetime, timedelta
 from typing import List, Optional
 
-from analytics.lib.counts import CountStat
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
-
-
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(start: datetime, end: datetime, frequency: str,
-               min_length: Optional[int]) -> List[datetime]:
+def time_range(start, end, frequency, min_length):
+    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
     verify_UTC(start)
     verify_UTC(end)
     if frequency == CountStat.HOUR:
@@ -20,7 +20,7 @@ def time_range(start: datetime, end: datetime, frequency: str,
         end = floor_to_day(end)
         step = timedelta(days=1)
     else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
+        raise AssertionError("Unknown frequency: %s" % (frequency,))
 
     times = []
     if min_length is not None:

analytics/management/commands/analyze_mit.py

@@ -1,22 +1,21 @@
-import datetime
-import logging
-import time
 from typing import Any, Dict
 
 from django.core.management.base import BaseCommand, CommandParser
-
+from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Message, Recipient
+import datetime
+import time
+import logging
 
-
-def compute_stats(log_level: int) -> None:
+def compute_stats(log_level):
+    # type: (int) -> None
     logger = logging.getLogger()
     logger.setLevel(log_level)
 
     one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
     mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
                                        recipient__type=Recipient.STREAM,
-                                       date_sent__gt=one_week_ago)
+                                       pub_date__gt=one_week_ago)
     for bot_sender_start in ["imap.", "rcmd.", "sys."]:
         mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
     # Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
@@ -27,15 +26,15 @@ def compute_stats(log_level: int) -> None:
                            "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                            "root@mit.edu", "nagios@mit.edu",
                            "www-data|local-realm@mit.edu"])
-    user_counts: Dict[str, Dict[str, int]] = {}
+    user_counts = {}  # type: Dict[str, Dict[str, int]]
     for m in mit_query.select_related("sending_client", "sender"):
         email = m.sender.email
         user_counts.setdefault(email, {})
         user_counts[email].setdefault(m.sending_client.name, 0)
         user_counts[email][m.sending_client.name] += 1
 
-    total_counts: Dict[str, int] = {}
-    total_user_counts: Dict[str, int] = {}
+    total_counts = {}  # type: Dict[str, int]
+    total_user_counts = {}  # type: Dict[str, int]
     for email, counts in user_counts.items():
         total_user_counts.setdefault(email, 0)
         for client_name, count in counts.items():
@@ -43,8 +42,8 @@ def compute_stats(log_level: int) -> None:
             total_counts[client_name] += count
             total_user_counts[email] += count
 
-    logging.debug("%40s | %10s | %s", "User", "Messages", "Percentage Zulip")
-    top_percents: Dict[int, float] = {}
+    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
+    top_percents = {}  # type: Dict[int, float]
     for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
         top_percents[size] = 0.0
     for i, email in enumerate(sorted(total_user_counts.keys(),
@@ -56,26 +55,28 @@ def compute_stats(log_level: int) -> None:
             if i < size:
                 top_percents[size] += (percent_zulip * 1.0 / size)
 
-        logging.debug("%40s | %10s | %s%%", email, total_user_counts[email],
-                      percent_zulip)
+        logging.debug("%40s | %10s | %s%%" % (email, total_user_counts[email],
+                                              percent_zulip))
 
     logging.info("")
     for size in sorted(top_percents.keys()):
-        logging.info("Top %6s | %s%%", size, round(top_percents[size], 1))
+        logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
 
     grand_total = sum(total_counts.values())
     print(grand_total)
-    logging.info("%15s | %s", "Client", "Percentage")
+    logging.info("%15s | %s" % ("Client", "Percentage"))
     for client in total_counts.keys():
-        logging.info("%15s | %s%%", client, round(100. * total_counts[client] / grand_total, 1))
+        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
 
 class Command(BaseCommand):
     help = "Compute statistics on MIT Zephyr usage."
 
-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
         parser.add_argument('--verbose', default=False, action='store_true')
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         level = logging.INFO
         if options["verbose"]:
             level = logging.DEBUG

analytics/management/commands/analyze_user_activity.py

@@ -1,14 +1,15 @@
-import datetime
 from typing import Any, Dict
 
-from django.core.management.base import BaseCommand, CommandParser
-
 from zerver.lib.statistics import seconds_usage_between
+
+from django.core.management.base import BaseCommand, CommandParser
 from zerver.models import UserProfile
+import datetime
+from django.utils.timezone import utc
 
-
-def analyze_activity(options: Dict[str, Any]) -> None:
-    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=datetime.timezone.utc)
+def analyze_activity(options):
+    # type: (Dict[str, Any]) -> None
+    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
     day_end = day_start + datetime.timedelta(days=options["duration"])
 
     user_profile_query = UserProfile.objects.all()
@@ -24,11 +25,11 @@ def analyze_activity(options: Dict[str, Any]) -> None:
             continue
 
         total_duration += duration
-        print(f"{user_profile.email:<37}{duration}")
+        print("%-*s%s" % (37, user_profile.email, duration,))
 
-    print(f"\nTotal Duration:                      {total_duration}")
-    print(f"\nTotal Duration in minutes:           {total_duration.total_seconds() / 60.}")
-    print(f"Total Duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}")
+    print("\nTotal Duration:                      %s" % (total_duration,))
+    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
+    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
 
 class Command(BaseCommand):
     help = """Report analytics of user activity on a per-user and realm basis.
@@ -46,11 +47,13 @@ Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""
 
-    def add_arguments(self, parser: CommandParser) -> None:
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
         parser.add_argument('--realm', action='store')
         parser.add_argument('--date', action='store', default="2013-09-06")
         parser.add_argument('--duration', action='store', default=1, type=int,
                             help="How many days to show usage information for")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         analyze_activity(options)

analytics/management/commands/check_analytics_state.py

@@ -1,86 +0,0 @@
-import os
-import time
-from datetime import timedelta
-from typing import Any, Dict
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.models import installation_epoch, last_successful_fill
-from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC
-from zerver.models import Realm
-
-states = {
-    0: "OK",
-    1: "WARNING",
-    2: "CRITICAL",
-    3: "UNKNOWN",
-}
-
-class Command(BaseCommand):
-    help = """Checks FillState table.
-
-    Run as a cron job that runs every hour."""
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        fill_state = self.get_fill_state()
-        status = fill_state['status']
-        message = fill_state['message']
-
-        state_file_path = "/var/lib/nagios_state/check-analytics-state"
-        state_file_tmp = state_file_path + "-tmp"
-
-        with open(state_file_tmp, "w") as f:
-            f.write(f"{int(time.time())}|{status}|{states[status]}|{message}\n")
-        os.rename(state_file_tmp, state_file_path)
-
-    def get_fill_state(self) -> Dict[str, Any]:
-        if not Realm.objects.exists():
-            return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
-
-        warning_unfilled_properties = []
-        critical_unfilled_properties = []
-        for property, stat in COUNT_STATS.items():
-            last_fill = last_successful_fill(property)
-            if last_fill is None:
-                last_fill = installation_epoch()
-            try:
-                verify_UTC(last_fill)
-            except TimezoneNotUTCException:
-                return {'status': 2, 'message': f'FillState not in UTC for {property}'}
-
-            if stat.frequency == CountStat.DAY:
-                floor_function = floor_to_day
-                warning_threshold = timedelta(hours=26)
-                critical_threshold = timedelta(hours=50)
-            else:  # CountStat.HOUR
-                floor_function = floor_to_hour
-                warning_threshold = timedelta(minutes=90)
-                critical_threshold = timedelta(minutes=150)
-
-            if floor_function(last_fill) != last_fill:
-                return {'status': 2, 'message': f'FillState not on {stat.frequency} boundary for {property}'}
-
-            time_to_last_fill = timezone_now() - last_fill
-            if time_to_last_fill > critical_threshold:
-                critical_unfilled_properties.append(property)
-            elif time_to_last_fill > warning_threshold:
-                warning_unfilled_properties.append(property)
-
-        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return {'status': 0, 'message': 'FillState looks fine.'}
-        if len(critical_unfilled_properties) == 0:
-            return {
-                'status': 1,
-                'message': 'Missed filling {} once.'.format(
-                    ', '.join(warning_unfilled_properties),
-                ),
-            }
-        return {
-            'status': 2,
-            'message': 'Missed filling {} once. Missed filling {} at least twice.'.format(
-                ', '.join(warning_unfilled_properties),
-                ', '.join(critical_unfilled_properties),
-            ),
-        }

analytics/management/commands/clear_analytics_tables.py

@@ -1,21 +1,26 @@
-from argparse import ArgumentParser
-from typing import Any
+import sys
 
-from django.core.management.base import BaseCommand, CommandError
+from argparse import ArgumentParser
+from django.db import connection
+from django.core.management.base import BaseCommand
 
 from analytics.lib.counts import do_drop_all_analytics_tables
 
+from typing import Any
 
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--force',
                             action='store_true',
                             help="Clear analytics tables.")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['force']:
             do_drop_all_analytics_tables()
         else:
-            raise CommandError("Would delete all data from analytics tables (!); use --force to do so.")
+            print("Would delete all data from analytics tables (!); use --force to do so.")
+            sys.exit(1)

analytics/management/commands/clear_single_stat.py

@@ -1,15 +1,18 @@
+import sys
+
 from argparse import ArgumentParser
+from django.db import connection
+from django.core.management.base import BaseCommand
+
+from analytics.lib.counts import do_drop_single_stat, COUNT_STATS
+
 from typing import Any
 
-from django.core.management.base import BaseCommand, CommandError
-
-from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
-
-
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--force',
                             action='store_true',
                             help="Actually do it.")
@@ -17,11 +20,14 @@ class Command(BaseCommand):
                             type=str,
                             help="The property of the stat to be cleared.")
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         property = options['property']
         if property not in COUNT_STATS:
-            raise CommandError(f"Invalid property: {property}")
+            print("Invalid property: %s" % (property,))
+            sys.exit(1)
         if not options['force']:
-            raise CommandError("No action taken. Use --force.")
+            print("No action taken. Use --force.")
+            sys.exit(1)
 
         do_drop_single_stat(property)

analytics/management/commands/client_activity.py

@@ -1,13 +1,13 @@
-import datetime
-from argparse import ArgumentParser
-from typing import Any, Optional
+from typing import Any
 
+from argparse import ArgumentParser
 from django.db.models import Count, QuerySet
 from django.utils.timezone import now as timezone_now
 
 from zerver.lib.management import ZulipBaseCommand
 from zerver.models import UserActivity
 
+import datetime
 
 class Command(ZulipBaseCommand):
     help = """Report rough client activity globally, for a realm, or for a user
@@ -18,16 +18,18 @@ Usage examples:
 ./manage.py client_activity --target realm --realm zulip
 ./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--target', dest='target', required=True, type=str,
                             help="'server' will calculate client activity of the entire server. "
                                  "'realm' will calculate client activity of realm. "
                                  "'user' will calculate client activity of the user.")
         parser.add_argument('--user', dest='user', type=str,
-                            help="The email address of the user you want to calculate activity.")
+                            help="The email adress of the user you want to calculate activity.")
         self.add_realm_args(parser)
 
-    def compute_activity(self, user_activity_objects: QuerySet) -> None:
+    def compute_activity(self, user_activity_objects):
+        # type: (QuerySet) -> None
         # Report data from the past week.
         #
         # This is a rough report of client activity because we inconsistently
@@ -54,10 +56,11 @@ Usage examples:
         counts.sort()
 
         for count in counts:
-            print(f"{count[1]:>25} {count[0]:15}")
+            print("%25s %15d" % (count[1], count[0]))
         print("Total:", total)
 
-    def handle(self, *args: Any, **options: Optional[str]) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
         realm = self.get_realm(options)
         if options["user"] is None:
             if options["target"] == "server" and realm is None:

analytics/management/commands/populate_analytics_db.py

@@ -1,6 +1,5 @@
-from datetime import timedelta
-from typing import Any, Dict, List, Mapping, Optional, Type
-from unittest import mock
+
+from argparse import ArgumentParser
 
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
@@ -8,19 +7,16 @@ from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    FillState,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-)
-from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, do_change_user_role
-from zerver.lib.create_user import create_user
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount, FillState
 from zerver.lib.timestamp import floor_to_day
-from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserProfile
+from zerver.models import Realm, UserProfile, Stream, Message, Client, \
+    RealmAuditLog
 
+from datetime import datetime, timedelta
+
+from six.moves import zip
+from typing import Any, Dict, List, Optional, Text, Type, Union, Mapping
 
 class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
@@ -28,10 +24,21 @@ class Command(BaseCommand):
     DAYS_OF_DATA = 100
     random_seed = 26
 
-    def generate_fixture_data(self, stat: CountStat, business_hours_base: float,
-                              non_business_hours_base: float, growth: float,
-                              autocorrelation: float, spikiness: float,
-                              holiday_rate: float=0, partial_sum: bool=False) -> List[int]:
+    def create_user(self, email, full_name, is_staff, date_joined, realm):
+        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
+        user = UserProfile.objects.create(
+            email=email, full_name=full_name, is_staff=is_staff,
+            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
+            api_key='42', date_joined=date_joined)
+        RealmAuditLog.objects.create(
+            realm=realm, modified_user=user, event_type='user_created',
+            event_time=user.date_joined)
+        return user
+
+    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
+                              growth, autocorrelation, spikiness, holiday_rate=0,
+                              partial_sum=False):
+        # type: (CountStat, float, float, float, float, float, float, bool) -> List[int]
         self.random_seed += 1
         return generate_time_series_data(
             days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
@@ -39,118 +46,47 @@ class Command(BaseCommand):
             autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
             frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
 
-    def handle(self, *args: Any, **options: Any) -> None:
-        # TODO: This should arguably only delete the objects
-        # associated with the "analytics" realm.
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         do_drop_all_analytics_tables()
-
-        # This also deletes any objects with this realm as a foreign key
+        # I believe this also deletes any objects with this realm as a foreign key
         Realm.objects.filter(string_id='analytics').delete()
 
-        # Because we just deleted a bunch of objects in the database
-        # directly (rather than deleting individual objects in Django,
-        # in which case our post_save hooks would have flushed the
-        # individual objects from memcached for us), we need to flush
-        # memcached in order to ensure deleted objects aren't still
-        # present in the memcached cache.
-        from zerver.apps import flush_cache
-        flush_cache(None)
-
         installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
         last_end_time = floor_to_day(timezone_now())
         realm = Realm.objects.create(
             string_id='analytics', name='Analytics', date_created=installation_time)
-        with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time):
-            shylock = create_user(
-                'shylock@analytics.ds',
-                'Shylock',
-                realm,
-                full_name='Shylock',
-                role=UserProfile.ROLE_REALM_ADMINISTRATOR
-            )
-        do_change_user_role(shylock, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None)
-        stream = Stream.objects.create(
-            name='all', realm=realm, date_created=installation_time)
-        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
-        stream.recipient = recipient
-        stream.save(update_fields=["recipient"])
+        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
 
-        # Subscribe shylock to the stream to avoid invariant failures.
-        # TODO: This should use subscribe_users_to_streams from populate_db.
-        subs = [
-            Subscription(recipient=recipient,
-                         user_profile=shylock,
-                         color=STREAM_ASSIGNMENT_COLORS[0]),
-        ]
-        Subscription.objects.bulk_create(subs)
-
-        def insert_fixture_data(stat: CountStat,
-                                fixture_data: Mapping[Optional[str], List[int]],
-                                table: Type[BaseCount]) -> None:
+        def insert_fixture_data(stat, fixture_data, table):
+            # type: (CountStat, Mapping[Optional[str], List[int]], Type[BaseCount]) -> None
             end_times = time_range(last_end_time, last_end_time, stat.frequency,
                                    len(list(fixture_data.values())[0]))
-            if table == InstallationCount:
-                id_args: Dict[str, Any] = {}
             if table == RealmCount:
                 id_args = {'realm': realm}
             if table == UserCount:
                 id_args = {'realm': realm, 'user': shylock}
-            if table == StreamCount:
-                id_args = {'stream': stream, 'realm': realm}
-
             for subgroup, values in fixture_data.items():
                 table.objects.bulk_create([
                     table(property=stat.property, subgroup=subgroup, end_time=end_time,
                           value=value, **id_args)
                     for end_time, value in zip(end_times, values) if value != 0])
 
-        stat = COUNT_STATS['1day_actives::day']
-        realm_data: Mapping[Optional[str], List[int]] = {
-            None: self.generate_fixture_data(stat, .08, .02, 3, .3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data: Mapping[Optional[str], List[int]] = {
-            None: self.generate_fixture_data(stat, .8, .2, 4, .3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
         stat = COUNT_STATS['realm_active_humans::day']
         realm_data = {
             None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
-        }
+        }  # type: Mapping[Optional[str], List[int]]
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 1, .3, 4, .5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        realm_data = {
-            'false': self.generate_fixture_data(stat, .1, .03, 3.5, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'false': self.generate_fixture_data(stat, 1, .3, 6, .8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
         FillState.objects.create(property=stat.property, end_time=last_end_time,
                                  state=FillState.DONE)
 
         stat = COUNT_STATS['messages_sent:is_bot:hour']
-        user_data: Mapping[Optional[str], List[int]] = {
-            'false': self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8, holiday_rate=.1),
-        }
+        user_data = {'false': self.generate_fixture_data(
+            stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}  # type: Mapping[Optional[str], List[int]]
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
                       'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {'false': self.generate_fixture_data(stat, 350, 150, 6, .6, 4),
-                             'true': self.generate_fixture_data(stat, 150, 150, 3, .4, 2)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
         FillState.objects.create(property=stat.property, end_time=last_end_time,
                                  state=FillState.DONE)
 
@@ -166,12 +102,6 @@ class Command(BaseCommand):
             'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
             'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            'public_stream': self.generate_fixture_data(stat, 300, 80, 5, .6, 4),
-            'private_stream': self.generate_fixture_data(stat, 70, 70, 5, .6, 4),
-            'private_message': self.generate_fixture_data(stat, 130, 50, 5, .6, 4),
-            'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, .6, 4)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
         FillState.objects.create(property=stat.property, end_time=last_end_time,
                                  state=FillState.DONE)
 
@@ -201,40 +131,7 @@ class Command(BaseCommand):
             unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
             long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            website.id: self.generate_fixture_data(stat, 300, 200, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, .6, 3),
-            android.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            API.id: self.generate_fixture_data(stat, 50, 50, 5, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, .6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3)}
-        insert_fixture_data(stat, installation_data, InstallationCount)
         FillState.objects.create(property=stat.property, end_time=last_end_time,
                                  state=FillState.DONE)
 
-        stat = COUNT_STATS['messages_in_stream:is_bot:day']
-        realm_data = {'false': self.generate_fixture_data(stat, 30, 5, 6, .6, 4),
-                      'true': self.generate_fixture_data(stat, 20, 2, 3, .2, 3)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data: Mapping[Optional[str], List[int]] = {
-            'false': self.generate_fixture_data(stat, 10, 7, 5, .6, 4),
-            'true': self.generate_fixture_data(stat, 5, 3, 2, .4, 2),
-        }
-        insert_fixture_data(stat, stream_data, StreamCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
-
-        stat = COUNT_STATS['messages_read::hour']
-        user_data = {
-            None: self.generate_fixture_data(stat, 7, 3, 2, .6, 8, holiday_rate=.1),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            None: self.generate_fixture_data(stat, 50, 35, 6, .6, 4)
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(property=stat.property, end_time=last_end_time,
-                                 state=FillState.DONE)
+        # TODO: messages_sent_to_stream:is_bot

analytics/management/commands/realm_stats.py

@@ -1,22 +1,15 @@
-import datetime
-from argparse import ArgumentParser
 from typing import Any, List
 
-from django.core.management.base import BaseCommand, CommandError
+from argparse import ArgumentParser
+import datetime
+import pytz
+
+from django.core.management.base import BaseCommand
 from django.db.models import Count
 from django.utils.timezone import now as timezone_now
 
-from zerver.models import (
-    Message,
-    Realm,
-    Recipient,
-    Stream,
-    Subscription,
-    UserActivity,
-    UserMessage,
-    UserProfile,
-    get_realm,
-)
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
+    Subscription, UserMessage, get_realm
 
 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
@@ -26,11 +19,13 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
 class Command(BaseCommand):
     help = "Generate statistics on realm activity."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def active_users(self, realm: Realm) -> List[UserProfile]:
+    def active_users(self, realm):
+        # type: (Realm) -> List[UserProfile]
         # Has been active (on the website, for now) in the last 7 days.
         activity_cutoff = timezone_now() - datetime.timedelta(days=7)
         return [activity.user_profile for activity in (
@@ -40,49 +35,59 @@ class Command(BaseCommand):
                                         query="/json/users/me/pointer",
                                         client__name="website"))]
 
-    def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
+    def messages_sent_by(self, user, days_ago):
+        # type: (UserProfile, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender=user, date_sent__gt=sent_time_cutoff).count()
+        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
 
-    def total_messages(self, realm: Realm, days_ago: int) -> int:
+    def total_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return Message.objects.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
+        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
-    def human_messages(self, realm: Realm, days_ago: int) -> int:
+    def human_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
 
-    def api_messages(self, realm: Realm, days_ago: int) -> int:
+    def api_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
 
-    def stream_messages(self, realm: Realm, days_ago: int) -> int:
+    def stream_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff,
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                      recipient__type=Recipient.STREAM).count()
 
-    def private_messages(self, realm: Realm, days_ago: int) -> int:
+    def private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
 
-    def group_private_messages(self, realm: Realm, days_ago: int) -> int:
+    def group_private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
         sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
-        return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
             recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
 
-    def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
+    def report_percentage(self, numerator, denominator, text):
+        # type: (float, float, str) -> None
         if not denominator:
             fraction = 0.0
         else:
             fraction = numerator / float(denominator)
-        print(f"{fraction * 100:.2f}% of", text)
+        print("%.2f%% of" % (fraction * 100,), text)
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]
             except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
         else:
             realms = Realm.objects.all()
 
@@ -93,26 +98,26 @@ class Command(BaseCommand):
             active_users = self.active_users(realm)
             num_active = len(active_users)
 
-            print(f"{num_active} active users ({len(user_profiles)} total)")
+            print("%d active users (%d total)" % (num_active, len(user_profiles)))
             streams = Stream.objects.filter(realm=realm).extra(
                 tables=['zerver_subscription', 'zerver_recipient'],
                 where=['zerver_subscription.recipient_id = zerver_recipient.id',
                        'zerver_recipient.type = 2',
                        'zerver_recipient.type_id = zerver_stream.id',
                        'zerver_subscription.active = true']).annotate(count=Count("name"))
-            print(f"{streams.count()} streams")
+            print("%d streams" % (streams.count(),))
 
             for days_ago in (1, 7, 30):
-                print(f"In last {days_ago} days, users sent:")
+                print("In last %d days, users sent:" % (days_ago,))
                 sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
                 for quantity in sorted(sender_quantities, reverse=True):
                     print(quantity, end=' ')
                 print("")
 
-                print(f"{self.stream_messages(realm, days_ago)} stream messages")
-                print(f"{self.private_messages(realm, days_ago)} one-on-one private messages")
-                print(f"{self.api_messages(realm, days_ago)} messages sent via the API")
-                print(f"{self.group_private_messages(realm, days_ago)} group private messages")
+                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
+                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
+                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
+                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
 
             num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
             self.report_percentage(num_notifications_enabled, num_active,
@@ -132,29 +137,29 @@ class Command(BaseCommand):
             starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
                                                   flags=UserMessage.flags.starred).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print("{} users have starred {} messages".format(
+            print("%d users have starred %d messages" % (
                 len(starrers), sum([elt["count"] for elt in starrers])))
 
             active_user_subs = Subscription.objects.filter(
                 user_profile__in=user_profiles, active=True)
 
             # Streams not in home view
-            non_home_view = active_user_subs.filter(is_muted=True).values(
+            non_home_view = active_user_subs.filter(in_home_view=False).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print("{} users have {} streams not in home view".format(
+            print("%d users have %d streams not in home view" % (
                 len(non_home_view), sum([elt["count"] for elt in non_home_view])))
 
             # Code block markup
             markup_messages = human_messages.filter(
                 sender__realm=realm, content__contains="~~~").values(
                 "sender").annotate(count=Count("sender"))
-            print("{} users have used code block markup on {} messages".format(
+            print("%d users have used code block markup on %s messages" % (
                 len(markup_messages), sum([elt["count"] for elt in markup_messages])))
 
             # Notifications for stream messages
-            notifications = active_user_subs.filter(desktop_notifications=True).values(
+            notifications = active_user_subs.filter(notifications=True).values(
                 "user_profile").annotate(count=Count("user_profile"))
-            print("{} users receive desktop notifications for {} streams".format(
+            print("%d users receive desktop notifications for %d streams" % (
                 len(notifications), sum([elt["count"] for elt in notifications])))
 
             print("")

analytics/management/commands/stream_stats.py

@@ -1,56 +1,43 @@
-from argparse import ArgumentParser
 from typing import Any
 
-from django.core.management.base import BaseCommand, CommandError
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
 from django.db.models import Q
-
-from zerver.models import Message, Realm, Recipient, Stream, Subscription, get_realm
-
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
 
 class Command(BaseCommand):
     help = "Generate statistics on the streams for a realm."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def handle(self, *args: Any, **options: str) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]
             except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
         else:
             realms = Realm.objects.all()
 
         for realm in realms:
+            print(realm.string_id)
+            print("------------")
+            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
             streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
-            # private stream count
-            private_count = 0
-            # public stream count
-            public_count = 0
+            invite_only_count = 0
             for stream in streams:
                 if stream.invite_only:
-                    private_count += 1
-                else:
-                    public_count += 1
-            print("------------")
-            print(realm.string_id, end=' ')
-            print("{:>10} {} public streams and".format("(", public_count), end=' ')
-            print(f"{private_count} private streams )")
-            print("------------")
-            print("{:>25} {:>15} {:>10} {:>12}".format("stream", "subscribers", "messages", "type"))
-
-            for stream in streams:
-                if stream.invite_only:
-                    stream_type = 'private'
-                else:
-                    stream_type = 'public'
-                print(f"{stream.name:>25}", end=' ')
+                    invite_only_count += 1
+                    continue
+                print("%25s" % (stream.name,), end=' ')
                 recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
-                print("{:10}".format(len(Subscription.objects.filter(recipient=recipient,
-                                                                     active=True))), end=' ')
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
                 num_messages = len(Message.objects.filter(recipient=recipient))
-                print(f"{num_messages:12}", end=' ')
-                print(f"{stream_type:>15}")
+                print("%12d" % (num_messages,))
+            print("%d invite-only streams" % (invite_only_count,))
             print("")

analytics/management/commands/update_analytics_counts.py

@@ -1,31 +1,34 @@
 import os
-import time
-from argparse import ArgumentParser
-from datetime import timezone
-from typing import Any, Dict
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from django.utils.dateparse import parse_datetime
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+import sys
 from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.remote_server import send_analytics_to_remote_server
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import Realm
 
+from argparse import ArgumentParser
+from datetime import timedelta
+import time
+
+from django.core.management.base import BaseCommand
+from django.utils.timezone import now as timezone_now
+from django.utils.timezone import utc as timezone_utc
+from django.utils.dateparse import parse_datetime
+from django.conf import settings
+
+from analytics.models import RealmCount, UserCount
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from zerver.lib.timestamp import floor_to_hour
+from zerver.models import UserProfile, Message, Realm
+
+from typing import Any, Dict
 
 class Command(BaseCommand):
     help = """Fills Analytics tables.
 
     Run as a cron job that runs every hour."""
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('--time', '-t',
                             type=str,
-                            help='Update stat tables from current state to'
-                                 '--time. Defaults to the current time.',
+                            help='Update stat tables from current state to --time. Defaults to the current time.',
                             default=timezone_now().isoformat())
         parser.add_argument('--utc',
                             action='store_true',
@@ -39,7 +42,8 @@ class Command(BaseCommand):
                             help="Print timing information to stdout.",
                             default=False)
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         try:
             os.mkdir(settings.ANALYTICS_LOCK_DIR)
         except OSError:
@@ -51,7 +55,8 @@ class Command(BaseCommand):
         finally:
             os.rmdir(settings.ANALYTICS_LOCK_DIR)
 
-    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
+    def run_update_analytics_counts(self, options):
+        # type: (Dict[str, Any]) -> None
         # installation_epoch relies on there being at least one realm; we
         # shouldn't run the analytics code if that condition isn't satisfied
         if not Realm.objects.exists():
@@ -60,18 +65,18 @@ class Command(BaseCommand):
 
         fill_to_time = parse_datetime(options['time'])
         if options['utc']:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
+            fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
         if fill_to_time.tzinfo is None:
             raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
 
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
+        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
 
         if options['stat'] is not None:
             stats = [COUNT_STATS[options['stat']]]
         else:
             stats = list(COUNT_STATS.values())
 
-        logger.info("Starting updating analytics counts through %s", fill_to_time)
+        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
         if options['verbose']:
             start = time.time()
             last = start
@@ -79,12 +84,10 @@ class Command(BaseCommand):
         for stat in stats:
             process_count_stat(stat, fill_to_time)
             if options['verbose']:
-                print(f"Updated {stat.property} in {time.time() - last:.3f}s")
+                print("Updated %s in %.3fs" % (stat.property, time.time() - last))
                 last = time.time()
 
         if options['verbose']:
-            print(f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s")
-        logger.info("Finished updating analytics counts through %s", fill_to_time)
-
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
-            send_analytics_to_remote_server()
+            print("Finished updating analytics counts through %s in %.3fs" %
+                  (fill_to_time, time.time() - start))
+        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))

analytics/management/commands/user_stats.py

@@ -1,42 +1,46 @@
-import datetime
 from argparse import ArgumentParser
+import datetime
+import pytz
 from typing import Any
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 
-from zerver.models import Message, Realm, Stream, UserProfile, get_realm
-
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
 
 class Command(BaseCommand):
     help = "Generate statistics on user activity."
 
-    def add_arguments(self, parser: ArgumentParser) -> None:
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
         parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                             help="realm to generate statistics for")
 
-    def messages_sent_by(self, user: UserProfile, week: int) -> int:
+    def messages_sent_by(self, user, week):
+        # type: (UserProfile, int) -> int
         start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
         end = timezone_now() - datetime.timedelta(days=week*7)
-        return Message.objects.filter(sender=user, date_sent__gt=start, date_sent__lte=end).count()
+        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
 
-    def handle(self, *args: Any, **options: Any) -> None:
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
         if options['realms']:
             try:
                 realms = [get_realm(string_id) for string_id in options['realms']]
             except Realm.DoesNotExist as e:
-                raise CommandError(e)
+                print(e)
+                exit(1)
         else:
             realms = Realm.objects.all()
 
         for realm in realms:
             print(realm.string_id)
             user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
-            print(f"{len(user_profiles)} users")
-            print(f"{len(Stream.objects.filter(realm=realm))} streams")
+            print("%d users" % (len(user_profiles),))
+            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
 
             for user_profile in user_profiles:
-                print(f"{user_profile.email:>35}", end=' ')
+                print("%35s" % (user_profile.email,), end=' ')
                 for week in range(10):
-                    print(f"{self.messages_sent_by(user_profile, week):5}", end=' ')
+                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
                 print("")

analytics/migrations/0001_initial.py

@@ -1,6 +1,8 @@
+# -*- coding: utf-8 -*-
+from django.db import models, migrations
 import django.db.models.deletion
 from django.conf import settings
-from django.db import migrations, models
+import zerver.lib.str_utils
 
 
 class Migration(migrations.Migration):
@@ -17,7 +19,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('info', models.CharField(max_length=1000)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='HuddleCount',
@@ -31,7 +33,7 @@ class Migration(migrations.Migration):
                 ('value', models.BigIntegerField()),
                 ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='InstallationCount',
@@ -43,7 +45,7 @@ class Migration(migrations.Migration):
                 ('value', models.BigIntegerField()),
                 ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='RealmCount',
@@ -57,7 +59,7 @@ class Migration(migrations.Migration):
                 ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
 
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='StreamCount',
@@ -71,7 +73,7 @@ class Migration(migrations.Migration):
                 ('value', models.BigIntegerField()),
                 ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
             name='UserCount',
@@ -85,26 +87,26 @@ class Migration(migrations.Migration):
                 ('value', models.BigIntegerField()),
                 ('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.AlterUniqueTogether(
             name='usercount',
-            unique_together={('user', 'property', 'end_time', 'interval')},
+            unique_together=set([('user', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='streamcount',
-            unique_together={('stream', 'property', 'end_time', 'interval')},
+            unique_together=set([('stream', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='realmcount',
-            unique_together={('realm', 'property', 'end_time', 'interval')},
+            unique_together=set([('realm', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='installationcount',
-            unique_together={('property', 'end_time', 'interval')},
+            unique_together=set([('property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='huddlecount',
-            unique_together={('huddle', 'property', 'end_time', 'interval')},
+            unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
         ),
     ]

analytics/migrations/0002_remove_huddlecount.py

@@ -1,4 +1,5 @@
-from django.db import migrations
+# -*- coding: utf-8 -*-
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -10,7 +11,7 @@ class Migration(migrations.Migration):
     operations = [
         migrations.AlterUniqueTogether(
             name='huddlecount',
-            unique_together=set(),
+            unique_together=set([]),
         ),
         migrations.RemoveField(
             model_name='huddlecount',

analytics/migrations/0003_fillstate.py

@@ -1,4 +1,6 @@
+# -*- coding: utf-8 -*-
 from django.db import migrations, models
+import zerver.lib.str_utils
 
 
 class Migration(migrations.Migration):
@@ -17,6 +19,6 @@ class Migration(migrations.Migration):
                 ('state', models.PositiveSmallIntegerField()),
                 ('last_modified', models.DateTimeField(auto_now=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.db import migrations, models
 
 

analytics/migrations/0005_alter_field_size.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from django.db import migrations, models
 
 

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -1,4 +1,5 @@
-from django.db import migrations
+# -*- coding: utf-8 -*-
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -10,18 +11,18 @@ class Migration(migrations.Migration):
     operations = [
         migrations.AlterUniqueTogether(
             name='installationcount',
-            unique_together={('property', 'subgroup', 'end_time', 'interval')},
+            unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='realmcount',
-            unique_together={('realm', 'property', 'subgroup', 'end_time', 'interval')},
+            unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='streamcount',
-            unique_together={('stream', 'property', 'subgroup', 'end_time', 'interval')},
+            unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
             name='usercount',
-            unique_together={('user', 'property', 'subgroup', 'end_time', 'interval')},
+            unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
     ]

analytics/migrations/0007_remove_interval.py

@@ -1,4 +1,6 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-16 20:50
+from django.conf import settings
 from django.db import migrations
 
 
@@ -11,7 +13,7 @@ class Migration(migrations.Migration):
     operations = [
         migrations.AlterUniqueTogether(
             name='installationcount',
-            unique_together={('property', 'subgroup', 'end_time')},
+            unique_together=set([('property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
             model_name='installationcount',
@@ -19,7 +21,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AlterUniqueTogether(
             name='realmcount',
-            unique_together={('realm', 'property', 'subgroup', 'end_time')},
+            unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
             model_name='realmcount',
@@ -27,7 +29,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AlterUniqueTogether(
             name='streamcount',
-            unique_together={('stream', 'property', 'subgroup', 'end_time')},
+            unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
             model_name='streamcount',
@@ -35,7 +37,7 @@ class Migration(migrations.Migration):
         ),
         migrations.AlterUniqueTogether(
             name='usercount',
-            unique_together={('user', 'property', 'subgroup', 'end_time')},
+            unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
             model_name='usercount',

analytics/migrations/0008_add_count_indexes.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-02-01 22:28
 from django.db import migrations
 
@@ -12,14 +13,14 @@ class Migration(migrations.Migration):
     operations = [
         migrations.AlterIndexTogether(
             name='realmcount',
-            index_together={('property', 'end_time')},
+            index_together=set([('property', 'end_time')]),
         ),
         migrations.AlterIndexTogether(
             name='streamcount',
-            index_together={('property', 'realm', 'end_time')},
+            index_together=set([('property', 'realm', 'end_time')]),
         ),
         migrations.AlterIndexTogether(
             name='usercount',
-            index_together={('property', 'realm', 'end_time')},
+            index_together=set([('property', 'realm', 'end_time')]),
         ),
     ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,9 +1,11 @@
-from django.db import migrations
-from django.db.backends.postgresql.schema import DatabaseSchemaEditor
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
+from django.db import migrations
 
 
-def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
+def delete_messages_sent_to_stream_stat(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
     UserCount = apps.get_model('analytics', 'UserCount')
     StreamCount = apps.get_model('analytics', 'StreamCount')
     RealmCount = apps.get_model('analytics', 'RealmCount')

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,9 +1,10 @@
-from django.db import migrations
-from django.db.backends.postgresql.schema import DatabaseSchemaEditor
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
+from django.db import migrations
 
-
-def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
+def clear_message_sent_by_message_type_values(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
     UserCount = apps.get_model('analytics', 'UserCount')
     StreamCount = apps.get_model('analytics', 'StreamCount')
     RealmCount = apps.get_model('analytics', 'RealmCount')

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,9 +1,11 @@
-from django.db import migrations
-from django.db.backends.postgresql.schema import DatabaseSchemaEditor
+# -*- coding: utf-8 -*-
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
+from django.db import migrations
 
 
-def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
+def clear_analytics_tables(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
     UserCount = apps.get_model('analytics', 'UserCount')
     StreamCount = apps.get_model('analytics', 'StreamCount')
     RealmCount = apps.get_model('analytics', 'RealmCount')

analytics/migrations/0012_add_on_delete.py

@@ -1,34 +0,0 @@
-# Generated by Django 1.11.6 on 2018-01-29 08:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0011_clear_analytics_tables'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='installationcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='realmcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='streamcount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-        migrations.AlterField(
-            model_name='usercount',
-            name='anomaly',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
-        ),
-    ]

analytics/migrations/0013_remove_anomaly.py

@@ -1,32 +0,0 @@
-# Generated by Django 1.11.18 on 2019-02-02 02:47
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0012_add_on_delete'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='installationcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='realmcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='streamcount',
-            name='anomaly',
-        ),
-        migrations.RemoveField(
-            model_name='usercount',
-            name='anomaly',
-        ),
-        migrations.DeleteModel(
-            name='Anomaly',
-        ),
-    ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -1,17 +0,0 @@
-# Generated by Django 1.11.26 on 2020-01-27 04:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0013_remove_anomaly'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='fillstate',
-            name='last_modified',
-        ),
-    ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,59 +0,0 @@
-from django.db import migrations
-from django.db.backends.postgresql.schema import DatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db.models import Count, Sum
-
-
-def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
-    """This is a preparatory migration for our Analytics tables.
-
-    The backstory is that Django's unique_together indexes do not properly
-    handle the subgroup=None corner case (allowing duplicate rows that have a
-    subgroup of None), which meant that in race conditions, rather than updating
-    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
-    create a duplicate row.
-
-    In the next migration, we'll add a proper constraint to fix this bug, but
-    we need to fix any existing problematic rows before we can add that constraint.
-
-    We fix this in an appropriate fashion for each type of CountStat object; mainly
-    this means deleting the extra rows, but for LoggingCountStat objects, we need to
-    additionally combine the sums.
-    """
-    count_tables = dict(realm=apps.get_model('analytics', 'RealmCount'),
-                        user=apps.get_model('analytics', 'UserCount'),
-                        stream=apps.get_model('analytics', 'StreamCount'),
-                        installation=apps.get_model('analytics', 'InstallationCount'))
-
-    for name, count_table in count_tables.items():
-        value = [name, 'property', 'end_time']
-        if name == 'installation':
-            value = ['property', 'end_time']
-        counts = count_table.objects.filter(subgroup=None).values(*value).annotate(
-            Count('id'), Sum('value')).filter(id__count__gt=1)
-
-        for count in counts:
-            count.pop('id__count')
-            total_value = count.pop('value__sum')
-            duplicate_counts = list(count_table.objects.filter(**count))
-            first_count = duplicate_counts[0]
-            if count['property'] in ["invites_sent::day", "active_users_log:is_bot:day"]:
-                # For LoggingCountStat objects, the right fix is to combine the totals;
-                # for other CountStat objects, we expect the duplicates to have the same value.
-                # And so all we need to do is delete them.
-                first_count.value = total_value
-                first_count.save()
-            to_cleanup = duplicate_counts[1:]
-            for duplicate_count in to_cleanup:
-                duplicate_count.delete()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0014_remove_fillstate_last_modified'),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_duplicate_counts,
-                             reverse_code=migrations.RunPython.noop),
-    ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -1,61 +0,0 @@
-# Generated by Django 2.2.10 on 2020-02-29 19:40
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('analytics', '0015_clear_duplicate_counts'),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name='installationcount',
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name='realmcount',
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name='streamcount',
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name='usercount',
-            unique_together=set(),
-        ),
-        migrations.AddConstraint(
-            model_name='installationcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=False), fields=('property', 'subgroup', 'end_time'), name='unique_installation_count'),
-        ),
-        migrations.AddConstraint(
-            model_name='installationcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=True), fields=('property', 'end_time'), name='unique_installation_count_null_subgroup'),
-        ),
-        migrations.AddConstraint(
-            model_name='realmcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=False), fields=('realm', 'property', 'subgroup', 'end_time'), name='unique_realm_count'),
-        ),
-        migrations.AddConstraint(
-            model_name='realmcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=True), fields=('realm', 'property', 'end_time'), name='unique_realm_count_null_subgroup'),
-        ),
-        migrations.AddConstraint(
-            model_name='streamcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=False), fields=('stream', 'property', 'subgroup', 'end_time'), name='unique_stream_count'),
-        ),
-        migrations.AddConstraint(
-            model_name='streamcount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=True), fields=('stream', 'property', 'end_time'), name='unique_stream_count_null_subgroup'),
-        ),
-        migrations.AddConstraint(
-            model_name='usercount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=False), fields=('user', 'property', 'subgroup', 'end_time'), name='unique_user_count'),
-        ),
-        migrations.AddConstraint(
-            model_name='usercount',
-            constraint=models.UniqueConstraint(condition=models.Q(subgroup__isnull=True), fields=('user', 'property', 'end_time'), name='unique_user_count_null_subgroup'),
-        ),
-    ]

analytics/models.py

@@ -1,32 +1,37 @@
-import datetime
-from typing import Optional
-
 from django.db import models
-from django.db.models import Q, UniqueConstraint
 
+from zerver.models import Realm, UserProfile, Stream, Recipient
+from zerver.lib.str_utils import ModelReprMixin
 from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, Stream, UserProfile
 
+import datetime
 
-class FillState(models.Model):
-    property: str = models.CharField(max_length=40, unique=True)
-    end_time: datetime.datetime = models.DateTimeField()
+from typing import Optional, Tuple, Union, Dict, Any, Text
+
+class FillState(ModelReprMixin, models.Model):
+    property = models.CharField(max_length=40, unique=True)  # type: Text
+    end_time = models.DateTimeField()  # type: datetime.datetime
 
     # Valid states are {DONE, STARTED}
     DONE = 1
     STARTED = 2
-    state: int = models.PositiveSmallIntegerField()
+    state = models.PositiveSmallIntegerField()  # type: int
 
-    def __str__(self) -> str:
-        return f"<FillState: {self.property} {self.end_time} {self.state}>"
+    last_modified = models.DateTimeField(auto_now=True)  # type: datetime.datetime
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime.datetime:
+def installation_epoch():
+    # type: () -> datetime.datetime
     earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
     return floor_to_day(earliest_realm_creation)
 
-def last_successful_fill(property: str) -> Optional[datetime.datetime]:
+def last_successful_fill(property):
+    # type: (str) -> Optional[datetime.datetime]
     fillstate = FillState.objects.filter(property=property).first()
     if fillstate is None:
         return None
@@ -34,98 +39,71 @@ def last_successful_fill(property: str) -> Optional[datetime.datetime]:
         return fillstate.end_time
     return fillstate.end_time - datetime.timedelta(hours=1)
 
-class BaseCount(models.Model):
+# would only ever make entries here by hand
+class Anomaly(ModelReprMixin, models.Model):
+    info = models.CharField(max_length=1000)  # type: Text
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<Anomaly: %s... %s>" % (self.info, self.id)
+
+class BaseCount(ModelReprMixin, models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property: str = models.CharField(max_length=32)
-    subgroup: Optional[str] = models.CharField(max_length=16, null=True)
-    end_time: datetime.datetime = models.DateTimeField()
-    value: int = models.BigIntegerField()
+    property = models.CharField(max_length=32)  # type: Text
+    subgroup = models.CharField(max_length=16, null=True)  # type: Optional[Text]
+    end_time = models.DateTimeField()  # type: datetime.datetime
+    value = models.BigIntegerField()  # type: int
+    anomaly = models.ForeignKey(Anomaly, null=True)  # type: Optional[Anomaly]
 
-    class Meta:
+    class Meta(object):
         abstract = True
 
 class InstallationCount(BaseCount):
 
-    class Meta:
-        # Handles invalid duplicate InstallationCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name='unique_installation_count'),
-            UniqueConstraint(
-                fields=["property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name='unique_installation_count_null_subgroup'),
-        ]
+    class Meta(object):
+        unique_together = ("property", "subgroup", "end_time")
 
-    def __str__(self) -> str:
-        return f"<InstallationCount: {self.property} {self.subgroup} {self.value}>"
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
 
 class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
-        # Handles invalid duplicate RealmCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["realm", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name='unique_realm_count'),
-            UniqueConstraint(
-                fields=["realm", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name='unique_realm_count_null_subgroup'),
-        ]
+    class Meta(object):
+        unique_together = ("realm", "property", "subgroup", "end_time")
         index_together = ["property", "end_time"]
 
-    def __str__(self) -> str:
-        return f"<RealmCount: {self.realm} {self.property} {self.subgroup} {self.value}>"
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
 
 class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    user = models.ForeignKey(UserProfile)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
-        # Handles invalid duplicate UserCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["user", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name='unique_user_count'),
-            UniqueConstraint(
-                fields=["user", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name='unique_user_count_null_subgroup'),
-        ]
+    class Meta(object):
+        unique_together = ("user", "property", "subgroup", "end_time")
         # This index dramatically improves the performance of
         # aggregating from users to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> str:
-        return f"<UserCount: {self.user} {self.property} {self.subgroup} {self.value}>"
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
 
 class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    stream = models.ForeignKey(Stream)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
-        # Handles invalid duplicate StreamCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["stream", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name='unique_stream_count'),
-            UniqueConstraint(
-                fields=["stream", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name='unique_stream_count_null_subgroup'),
-        ]
+    class Meta(object):
+        unique_together = ("stream", "property", "subgroup", "end_time")
         # This index dramatically improves the performance of
         # aggregating from streams to realms
         index_together = ["property", "realm", "end_time"]
 
-    def __str__(self) -> str:
-        return f"<StreamCount: {self.stream} {self.property} {self.subgroup} {self.value} {self.id}>"
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)

analytics/tests/test_fixtures.py

@@ -1,11 +1,12 @@
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
 from zerver.lib.test_classes import ZulipTestCase
 
+from analytics.lib.counts import CountStat
+from analytics.lib.fixtures import generate_time_series_data
 
 # A very light test suite; the code being tested is not run in production.
 class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self) -> None:
+    def test_deterministic_settings(self):
+        # type: () -> None
         # test basic business_hour / non_business_hour calculation
         # test we get an array of the right length with frequency=CountStat.DAY
         data = generate_time_series_data(

analytics/tests/test_views.py

@@ -1,90 +1,49 @@
-from datetime import datetime, timedelta, timezone
-from typing import List, Optional
-from unittest import mock
-
-import ujson
-from django.http import HttpResponse
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, UserCount, last_successful_fill
-from analytics.views import rewrite_client_arrays, sort_by_totals, sort_client_labels
-from corporate.models import get_customer_by_realm
-from zerver.lib.actions import do_create_multiuse_invite_link, do_send_realm_reactivation_email
+from django.utils.timezone import get_fixed_timezone, utc
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import reset_emails_in_zulip_realm
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client, MultiuseInvite, PreregistrationUser, get_realm
+from zerver.lib.timestamp import ceiling_to_hour, ceiling_to_day, \
+    datetime_to_timestamp
+from zerver.models import Realm, UserProfile, Client, get_realm
 
+from analytics.lib.counts import CountStat, COUNT_STATS
+from analytics.lib.time_utils import time_range
+from analytics.models import RealmCount, UserCount, BaseCount, \
+    FillState, last_successful_fill
+from analytics.views import stats, get_chart_data, sort_by_totals, \
+    sort_client_labels, rewrite_client_arrays
+
+from datetime import datetime, timedelta
+import mock
+import ujson
+
+from typing import List, Dict, Optional
 
 class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
+    def test_stats(self):
+        # type: () -> None
         self.user = self.example_user('hamlet')
-        self.login_user(self.user)
+        self.login(self.user.email)
         result = self.client_get('/stats')
         self.assertEqual(result.status_code, 200)
         # Check that we get something back
         self.assert_in_response("Zulip analytics for", result)
 
-    def test_guest_user_cant_access_stats(self) -> None:
-        self.user = self.example_user('polonius')
-        self.login_user(self.user)
-        result = self.client_get('/stats')
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-        result = self.client_get('/json/analytics/chart_data')
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-    def test_stats_for_realm(self) -> None:
-        user = self.example_user('hamlet')
-        self.login_user(user)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user('hamlet')
-        user.is_staff = True
-        user.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/realm/not_existing_realm/')
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get('/stats/realm/zulip/')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user = self.example_user('hamlet')
-        self.login_user(user)
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user('hamlet')
-        user.is_staff = True
-        user.save(update_fields=['is_staff'])
-
-        result = self.client_get('/stats/installation')
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
 class TestGetChartData(ZulipTestCase):
-    def setUp(self) -> None:
-        super().setUp()
+    def setUp(self):
+        # type: () -> None
         self.realm = get_realm('zulip')
         self.user = self.example_user('hamlet')
-        self.login_user(self.user)
+        self.login(self.user.email)
         self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
                                for i in range(4)]
         self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
                               for i in range(4)]
 
-    def data(self, i: int) -> List[int]:
+    def data(self, i):
+        # type: (int) -> List[int]
         return [0, 0, i, 0]
 
-    def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]],
-                    user_subgroups: List[str]) -> None:
+    def insert_data(self, stat, realm_subgroups, user_subgroups):
+        # type: (CountStat, List[Optional[str]], List[str]) -> None
         if stat.frequency == CountStat.HOUR:
             insert_time = self.end_times_hour[2]
             fill_time = self.end_times_hour[-1]
@@ -102,13 +61,10 @@ class TestGetChartData(ZulipTestCase):
             for i, subgroup in enumerate(user_subgroups)])
         FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
 
-    def test_number_of_humans(self) -> None:
+    def test_number_of_humans(self):
+        # type: () -> None
         stat = COUNT_STATS['realm_active_humans::day']
         self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'number_of_humans'})
         self.assert_json_success(result)
@@ -117,12 +73,13 @@ class TestGetChartData(ZulipTestCase):
             'msg': '',
             'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
             'frequency': CountStat.DAY,
-            'everyone': {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)},
+            'realm': {'human': self.data(100)},
             'display_order': None,
             'result': 'success',
         })
 
-    def test_messages_sent_over_time(self) -> None:
+    def test_messages_sent_over_time(self):
+        # type: () -> None
         stat = COUNT_STATS['messages_sent:is_bot:hour']
         self.insert_data(stat, ['true', 'false'], ['false'])
         result = self.client_get('/json/analytics/chart_data',
@@ -133,13 +90,14 @@ class TestGetChartData(ZulipTestCase):
             'msg': '',
             'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
             'frequency': CountStat.HOUR,
-            'everyone': {'bot': self.data(100), 'human': self.data(101)},
+            'realm': {'bot': self.data(100), 'human': self.data(101)},
             'user': {'bot': self.data(0), 'human': self.data(200)},
             'display_order': None,
             'result': 'success',
         })
 
-    def test_messages_sent_by_message_type(self) -> None:
+    def test_messages_sent_by_message_type(self):
+        # type: () -> None
         stat = COUNT_STATS['messages_sent:message_type:day']
         self.insert_data(stat, ['public_stream', 'private_message'],
                          ['public_stream', 'private_stream'])
@@ -151,15 +109,16 @@ class TestGetChartData(ZulipTestCase):
             'msg': '',
             'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
             'frequency': CountStat.DAY,
-            'everyone': {'Public streams': self.data(100), 'Private streams': self.data(0),
-                         'Private messages': self.data(101), 'Group private messages': self.data(0)},
+            'realm': {'Public streams': self.data(100), 'Private streams': self.data(0),
+                      'Private messages': self.data(101), 'Group private messages': self.data(0)},
             'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
                      'Private messages': self.data(0), 'Group private messages': self.data(0)},
             'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
             'result': 'success',
         })
 
-    def test_messages_sent_by_client(self) -> None:
+    def test_messages_sent_by_client(self):
+        # type: () -> None
         stat = COUNT_STATS['messages_sent:client:day']
         client1 = Client.objects.create(name='client 1')
         client2 = Client.objects.create(name='client 2')
@@ -175,82 +134,57 @@ class TestGetChartData(ZulipTestCase):
             'msg': '',
             'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
             'frequency': CountStat.DAY,
-            'everyone': {'client 4': self.data(100), 'client 3': self.data(101),
-                         'client 2': self.data(102)},
+            'realm': {'client 4': self.data(100), 'client 3': self.data(101),
+                      'client 2': self.data(102)},
             'user': {'client 3': self.data(200), 'client 1': self.data(201)},
             'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
             'result': 'success',
         })
 
-    def test_messages_read_over_time(self) -> None:
-        stat = COUNT_STATS['messages_read::hour']
-        self.insert_data(stat, [None], [])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_read_over_time'})
-        self.assert_json_success(result)
-        data = result.json()
-        self.assertEqual(data, {
-            'msg': '',
-            'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-            'frequency': CountStat.HOUR,
-            'everyone': {'read': self.data(100)},
-            'user': {'read': self.data(0)},
-            'display_order': None,
-            'result': 'success',
-        })
-
-    def test_include_empty_subgroups(self) -> None:
+    def test_include_empty_subgroups(self):
+        # type: () -> None
         FillState.objects.create(
-            property='realm_active_humans::day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
+            property='realm_active_humans::day', end_time=self.end_times_day[0], state=FillState.DONE)
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'number_of_humans'})
         self.assert_json_success(result)
         data = result.json()
-        self.assertEqual(data['everyone'], {"_1day": [0], "_15day": [0], "all_time": [0]})
+        self.assertEqual(data['realm'], {'human': [0]})
         self.assertFalse('user' in data)
 
         FillState.objects.create(
-            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0],
-            state=FillState.DONE)
+            property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0], state=FillState.DONE)
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'messages_sent_over_time'})
         self.assert_json_success(result)
         data = result.json()
-        self.assertEqual(data['everyone'], {'human': [0], 'bot': [0]})
+        self.assertEqual(data['realm'], {'human': [0], 'bot': [0]})
         self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
 
         FillState.objects.create(
-            property='messages_sent:message_type:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
+            property='messages_sent:message_type:day', end_time=self.end_times_day[0], state=FillState.DONE)
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'messages_sent_by_message_type'})
         self.assert_json_success(result)
         data = result.json()
-        self.assertEqual(data['everyone'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
+        self.assertEqual(data['realm'], {
+            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
         self.assertEqual(data['user'], {
-            'Public streams': [0], 'Private streams': [0],
-            'Private messages': [0], 'Group private messages': [0]})
+            'Public streams': [0], 'Private streams': [0], 'Private messages': [0], 'Group private messages': [0]})
 
         FillState.objects.create(
-            property='messages_sent:client:day', end_time=self.end_times_day[0],
-            state=FillState.DONE)
+            property='messages_sent:client:day', end_time=self.end_times_day[0], state=FillState.DONE)
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'messages_sent_by_client'})
         self.assert_json_success(result)
         data = result.json()
-        self.assertEqual(data['everyone'], {})
+        self.assertEqual(data['realm'], {})
         self.assertEqual(data['user'], {})
 
-    def test_start_and_end(self) -> None:
+    def test_start_and_end(self):
+        # type: () -> None
         stat = COUNT_STATS['realm_active_humans::day']
         self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
         end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
 
         # valid start and end
@@ -261,7 +195,7 @@ class TestGetChartData(ZulipTestCase):
         self.assert_json_success(result)
         data = result.json()
         self.assertEqual(data['end_times'], end_time_timestamps[1:3])
-        self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})
+        self.assertEqual(data['realm'], {'human': [0, 100]})
 
         # start later then end
         result = self.client_get('/json/analytics/chart_data',
@@ -270,13 +204,10 @@ class TestGetChartData(ZulipTestCase):
                                   'end': end_time_timestamps[1]})
         self.assert_json_error_contains(result, 'Start time is later than')
 
-    def test_min_length(self) -> None:
+    def test_min_length(self):
+        # type: () -> None
         stat = COUNT_STATS['realm_active_humans::day']
         self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['1day_actives::day']
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS['active_users_audit:is_bot:day']
-        self.insert_data(stat, ['false'], [])
         # test min_length is too short to change anything
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'number_of_humans',
@@ -284,7 +215,7 @@ class TestGetChartData(ZulipTestCase):
         self.assert_json_success(result)
         data = result.json()
         self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
-        self.assertEqual(data['everyone'], {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)})
+        self.assertEqual(data['realm'], {'human': self.data(100)})
         # test min_length larger than filled data
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'number_of_humans',
@@ -293,367 +224,31 @@ class TestGetChartData(ZulipTestCase):
         data = result.json()
         end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
         self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(data['everyone'], {'_1day': [0]+self.data(100), '_15day': [0]+self.data(100), 'all_time': [0]+self.data(100)})
+        self.assertEqual(data['realm'], {'human': [0]+self.data(100)})
 
-    def test_non_existent_chart(self) -> None:
+    def test_non_existent_chart(self):
+        # type: () -> None
         result = self.client_get('/json/analytics/chart_data',
                                  {'chart_name': 'does_not_exist'})
         self.assert_json_error_contains(result, 'Unknown chart name')
 
-    def test_analytics_not_running(self) -> None:
-        realm = get_realm("zulip")
-
-        self.assertEqual(FillState.objects.count(), 0)
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
+    def test_analytics_not_running(self):
+        # type: () -> None
+        # try to get data for a valid chart, but before we've put anything in the database
+        # (e.g. before update_analytics_counts has been run)
         with mock.patch('logging.warning'):
             result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'messages_sent_over_time'})
+                                     {'chart_name': 'number_of_humans'})
         self.assert_json_error_contains(result, 'No analytics data available')
 
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(hours=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=5)
-        fill_state = FillState.objects.create(property='messages_sent:is_bot:hour', end_time=end_time,
-                                              state=FillState.DONE)
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=2)
-        fill_state.end_time = end_time
-        fill_state.save(update_fields=["end_time"])
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get('/json/analytics/chart_data',
-                                 {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with mock.patch('logging.warning'):
-            result = self.client_get('/json/analytics/chart_data',
-                                     {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_error_contains(result, 'No analytics data available')
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get('/json/analytics/chart_data', {'chart_name': 'messages_sent_over_time'})
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user = self.example_user('hamlet')
-        self.login_user(user)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user('hamlet')
-        user.is_staff = True
-        user.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, 'Invalid organization', 400)
-
-        result = self.client_get('/json/analytics/chart_data/realm/zulip',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user = self.example_user('hamlet')
-        self.login_user(user)
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user('hamlet')
-        user.is_staff = True
-        user.save(update_fields=['is_staff'])
-        stat = COUNT_STATS['realm_active_humans::day']
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get('/json/analytics/chart_data/installation',
-                                 {'chart_name': 'number_of_humans'})
-        self.assert_json_success(result)
-
-class TestSupportEndpoint(ZulipTestCase):
-    def test_search(self) -> None:
-        reset_emails_in_zulip_realm()
-
-        def check_hamlet_user_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>',
-                                             '<b>Email</b>: hamlet@zulip.com', '<b>Is active</b>: True<br>',
-                                             '<b>Admins</b>: desdemona@zulip.com, iago@zulip.com\n',
-                                             'class="copy-button" data-copytext="desdemona@zulip.com, iago@zulip.com"',
-                                             ], result)
-
-        def check_zulip_realm_query_result(result: HttpResponse) -> None:
-            zulip_realm = get_realm("zulip")
-            self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"',
-                                             'Zulip Dev</h3>',
-                                             '<option value="1" selected>Self Hosted</option>',
-                                             '<option value="2" >Limited</option>',
-                                             'input type="number" name="discount" value="None"',
-                                             '<option value="active" selected>Active</option>',
-                                             '<option value="deactivated" >Deactivated</option>',
-                                             'scrub-realm-button">',
-                                             'data-string-id="zulip"'], result)
-
-        def check_lear_realm_query_result(result: HttpResponse) -> None:
-            lear_realm = get_realm("lear")
-            self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{lear_realm.id}"',
-                                             'Lear &amp; Co.</h3>',
-                                             '<option value="1" selected>Self Hosted</option>',
-                                             '<option value="2" >Limited</option>',
-                                             'input type="number" name="discount" value="None"',
-                                             '<option value="active" selected>Active</option>',
-                                             '<option value="deactivated" >Deactivated</option>',
-                                             'scrub-realm-button">',
-                                             'data-string-id="lear"'], result)
-
-        def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: bool=False) -> None:
-            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
-                                             f'<b>Email</b>: {email}',
-                                             ], result)
-            if invite:
-                self.assert_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3',
-                                                 '<b>Status</b>: Link has never been clicked'], result)
-                self.assert_in_success_response([], result)
-            else:
-                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day',
-                                                 '<b>Status</b>: Link has never been clicked'], result)
-
-        def check_realm_creation_query_result(result: HttpResponse, email: str) -> None:
-            self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
-                                             '<span class="label">realm creation</span>\n',
-                                             '<b>Link</b>: http://testserver/accounts/do_confirm/',
-                                             '<b>Expires in</b>: 1\xa0day<br>\n',
-                                             ], result)
-
-        def check_multiuse_invite_link_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">multiuse invite</span>\n',
-                                             '<b>Link</b>: http://zulip.testserver/join/',
-                                             '<b>Expires in</b>: 1\xa0week, 3',
-                                             ], result)
-
-        def check_realm_reactivation_link_query_result(result: HttpResponse) -> None:
-            self.assert_in_success_response(['<span class="label">realm reactivation</span>\n',
-                                             '<b>Link</b>: http://zulip.testserver/reactivate/',
-                                             '<b>Expires in</b>: 1\xa0day',
-                                             ], result)
-
-        self.login('cordelia')
-
-        result = self.client_get("/activity/support")
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login('iago')
-
-        result = self.client_get("/activity/support")
-        self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result)
-
-        result = self.client_get("/activity/support", {"q": "hamlet@zulip.com"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "lear"})
-        check_lear_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "http://lear.testserver"})
-        check_lear_realm_query_result(result)
-
-        with self.settings(REALM_HOSTS={'zulip': 'localhost'}):
-            result = self.client_get("/activity/support", {"q": "http://localhost"})
-            check_zulip_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "hamlet@zulip.com, lear"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        result = self.client_get("/activity/support", {"q": "lear, Hamlet <hamlet@zulip.com>"})
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        self.client_post('/accounts/home/', {'email': self.nonreg_email("test")})
-        self.login('iago')
-        result = self.client_get("/activity/support", {"q": self.nonreg_email("test")})
-        check_preregistration_user_query_result(result, self.nonreg_email("test"))
-        check_zulip_realm_query_result(result)
-
-        stream_ids = [self.get_stream_id("Denmark")]
-        invitee_emails = [self.nonreg_email("test1")]
-        self.client_post("/json/invites", {"invitee_emails": invitee_emails,
-                                           "stream_ids": ujson.dumps(stream_ids),
-                                           "invite_as": PreregistrationUser.INVITE_AS['MEMBER']})
-        result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")})
-        check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
-        check_zulip_realm_query_result(result)
-
-        email = self.nonreg_email('alice')
-        self.client_post('/new/', {'email': email})
-        result = self.client_get("/activity/support", {"q": email})
-        check_realm_creation_query_result(result, email)
-
-        do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1)
-        result = self.client_get("/activity/support", {"q": "zulip"})
-        check_multiuse_invite_link_query_result(result)
-        check_zulip_realm_query_result(result)
-        MultiuseInvite.objects.all().delete()
-
-        do_send_realm_reactivation_email(get_realm("zulip"))
-        result = self.client_get("/activity/support", {"q": "zulip"})
-        check_realm_reactivation_link_query_result(result)
-        check_zulip_realm_query_result(result)
-
-    def test_change_plan_type(self) -> None:
-        cordelia = self.example_user('cordelia')
-        self.login_user(cordelia)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        with mock.patch("analytics.views.do_change_plan_type") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "2"})
-            m.assert_called_once_with(get_realm("zulip"), 2)
-            self.assert_in_success_response(["Plan type of Zulip Dev changed from self hosted to limited"], result)
-
-    def test_attach_discount(self) -> None:
-        cordelia = self.example_user('cordelia')
-        lear_realm = get_realm('lear')
-        self.login_user(cordelia)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login('iago')
-
-        with mock.patch("analytics.views.attach_discount_to_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"})
-            m.assert_called_once_with(get_realm("lear"), 25)
-            self.assert_in_success_response(["Discount of Lear &amp; Co. changed to 25 from None"], result)
-
-    def test_change_sponsorship_status(self) -> None:
-        lear_realm = get_realm("lear")
-        self.assertIsNone(get_customer_by_realm(lear_realm))
-
-        cordelia = self.example_user('cordelia')
-        self.login_user(cordelia)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}",
-                                                        "sponsorship_pending": "true"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}",
-                                                        "sponsorship_pending": "true"})
-        self.assert_in_success_response(["Lear &amp; Co. marked as pending sponsorship."], result)
-        customer = get_customer_by_realm(lear_realm)
-        assert(customer is not None)
-        self.assertTrue(customer.sponsorship_pending)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}",
-                                                        "sponsorship_pending": "false"})
-        self.assert_in_success_response(["Lear &amp; Co. is no longer pending sponsorship."], result)
-        customer = get_customer_by_realm(lear_realm)
-        assert(customer is not None)
-        self.assertFalse(customer.sponsorship_pending)
-
-    def test_activate_or_deactivate_realm(self) -> None:
-        cordelia = self.example_user('cordelia')
-        lear_realm = get_realm('lear')
-        self.login_user(cordelia)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login('iago')
-
-        with mock.patch("analytics.views.do_deactivate_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"})
-            m.assert_called_once_with(lear_realm, self.example_user("iago"))
-            self.assert_in_success_response(["Lear &amp; Co. deactivated"], result)
-
-        with mock.patch("analytics.views.do_send_realm_reactivation_email") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "status": "active"})
-            m.assert_called_once_with(lear_realm)
-            self.assert_in_success_response(["Realm reactivation email sent to admins of Lear"], result)
-
-    def test_scrub_realm(self) -> None:
-        cordelia = self.example_user('cordelia')
-        lear_realm = get_realm('lear')
-        self.login_user(cordelia)
-
-        result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"})
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login('iago')
-
-        with mock.patch("analytics.views.do_scrub_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}", "scrub_realm": "scrub_realm"})
-            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
-            self.assert_in_success_response(["Lear &amp; Co. scrubbed"], result)
-
-        with mock.patch("analytics.views.do_scrub_realm") as m:
-            with self.assertRaises(AssertionError):
-                result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
-            m.assert_not_called()
-
 class TestGetChartDataHelpers(ZulipTestCase):
     # last_successful_fill is in analytics/models.py, but get_chart_data is
     # the only function that uses it at the moment
-    def test_last_successful_fill(self) -> None:
+    def test_last_successful_fill(self):
+        # type: () -> None
         self.assertIsNone(last_successful_fill('non-existant'))
-        a_time = datetime(2016, 3, 14, 19, tzinfo=timezone.utc)
-        one_hour_before = datetime(2016, 3, 14, 18, tzinfo=timezone.utc)
+        a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
+        one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
         fillstate = FillState.objects.create(property='property', end_time=a_time,
                                              state=FillState.DONE)
         self.assertEqual(last_successful_fill('property'), a_time)
@@ -661,24 +256,27 @@ class TestGetChartDataHelpers(ZulipTestCase):
         fillstate.save()
         self.assertEqual(last_successful_fill('property'), one_hour_before)
 
-    def test_sort_by_totals(self) -> None:
-        empty: List[int] = []
+    def test_sort_by_totals(self):
+        # type: () -> None
+        empty = []  # type: List[int]
         value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
         self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
 
-    def test_sort_client_labels(self) -> None:
-        data = {'everyone': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
+    def test_sort_client_labels(self):
+        # type: () -> None
+        data = {'realm': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
                 'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
         self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
 
 class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
+    def test_time_range(self):
+        # type: () -> None
         HOUR = timedelta(hours=1)
         DAY = timedelta(days=1)
 
-        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
-        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
-        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
+        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
+        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
+        floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
 
         # test start == end
         self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
@@ -698,7 +296,8 @@ class TestTimeRange(ZulipTestCase):
                          [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
 
 class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
+    def test_map_arrays(self):
+        # type: () -> None
         a = {'desktop app 1.0': [1, 2, 3],
              'desktop app 2.0': [10, 12, 13],
              'desktop app 3.0': [21, 22, 23],

analytics/urls.py

@@ -1,34 +1,20 @@
-from django.conf.urls import include
-from django.urls import path
+from django.conf.urls import url, include
+from zerver.lib.rest import rest_dispatch
 
 import analytics.views
-from zerver.lib.rest import rest_dispatch
 
 i18n_urlpatterns = [
     # Server admin (user_profile.is_staff) visible stats pages
-    path('activity', analytics.views.get_activity,
-         name='analytics.views.get_activity'),
-    path('activity/support', analytics.views.support,
-         name='analytics.views.support'),
-    path('realm_activity/<str:realm_str>/', analytics.views.get_realm_activity,
-         name='analytics.views.get_realm_activity'),
-    path('user_activity/<str:email>/', analytics.views.get_user_activity,
-         name='analytics.views.get_user_activity'),
-
-    path('stats/realm/<str:realm_str>/', analytics.views.stats_for_realm,
-         name='analytics.views.stats_for_realm'),
-    path('stats/installation', analytics.views.stats_for_installation,
-         name='analytics.views.stats_for_installation'),
-    path('stats/remote/<int:remote_server_id>/installation',
-         analytics.views.stats_for_remote_installation,
-         name='analytics.views.stats_for_remote_installation'),
-    path('stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/',
-         analytics.views.stats_for_remote_realm,
-         name='analytics.views.stats_for_remote_realm'),
+    url(r'^activity$', analytics.views.get_activity,
+        name='analytics.views.get_activity'),
+    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
+        name='analytics.views.get_realm_activity'),
+    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
+        name='analytics.views.get_user_activity'),
 
     # User-visible stats page
-    path('stats', analytics.views.stats,
-         name='analytics.views.stats'),
+    url(r'^stats$', analytics.views.stats,
+        name='analytics.views.stats'),
 ]
 
 # These endpoints are a part of the API (V1), which uses:
@@ -41,22 +27,13 @@ i18n_urlpatterns = [
 # All of these paths are accessed by either a /json or /api prefix
 v1_api_and_json_patterns = [
     # get data for the graphs at /stats
-    path('analytics/chart_data', rest_dispatch,
-         {'GET': 'analytics.views.get_chart_data'}),
-    path('analytics/chart_data/realm/<str:realm_str>', rest_dispatch,
-         {'GET': 'analytics.views.get_chart_data_for_realm'}),
-    path('analytics/chart_data/installation', rest_dispatch,
-         {'GET': 'analytics.views.get_chart_data_for_installation'}),
-    path('analytics/chart_data/remote/<int:remote_server_id>/installation', rest_dispatch,
-         {'GET': 'analytics.views.get_chart_data_for_remote_installation'}),
-    path('analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>',
-         rest_dispatch,
-         {'GET': 'analytics.views.get_chart_data_for_remote_realm'}),
+    url(r'^analytics/chart_data$', rest_dispatch,
+        {'GET': 'analytics.views.get_chart_data'}),
 ]
 
 i18n_urlpatterns += [
-    path('api/v1/', include(v1_api_and_json_patterns)),
-    path('json/', include(v1_api_and_json_patterns)),
+    url(r'^api/v1/', include(v1_api_and_json_patterns)),
+    url(r'^json/', include(v1_api_and_json_patterns)),
 ]
 
 urlpatterns = i18n_urlpatterns

babel.config.js

@@ -1,18 +0,0 @@
-module.exports = {
-    presets: [
-        [
-            "@babel/preset-env",
-            {
-                corejs: 3,
-                loose: true, // Loose mode for…of loops are 5× faster in Firefox
-                useBuiltIns: "usage",
-            },
-        ],
-        "@babel/typescript",
-    ],
-    plugins: [
-        "@babel/proposal-class-properties",
-        ["@babel/plugin-proposal-unicode-property-regex", {useUnicodeFlag: false}],
-    ],
-    sourceType: "unambiguous",
-};

confirmation/__init__.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
 # Permission is hereby granted, free of charge, to any person obtaining a

confirmation/migrations/0001_initial.py

@@ -1,5 +1,6 @@
+# -*- coding: utf-8 -*-
+from django.db import models, migrations
 import django.db.models.deletion
-from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

confirmation/migrations/0002_realmcreationkey.py

@@ -1,5 +1,6 @@
+# -*- coding: utf-8 -*-
+from django.db import models, migrations
 import django.utils.timezone
-from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

confirmation/migrations/0003_emailchangeconfirmation.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-17 09:16
 from django.db import migrations
 

confirmation/migrations/0004_remove_confirmationmanager.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.11.2 on 2017-07-08 04:23
 from django.db import migrations, models
 

confirmation/migrations/0005_confirmation_realm.py

@@ -1,19 +0,0 @@
-# Generated by Django 1.11.6 on 2017-11-30 00:13
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('zerver', '0124_stream_enable_notifications'),
-        ('confirmation', '0004_remove_confirmationmanager'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='confirmation',
-            name='realm',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm'),
-        ),
-    ]

confirmation/migrations/0006_realmcreationkey_presume_email_valid.py

@@ -1,18 +0,0 @@
-# Generated by Django 1.11.6 on 2018-01-29 18:39
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('confirmation', '0005_confirmation_realm'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='realmcreationkey',
-            name='presume_email_valid',
-            field=models.BooleanField(default=False),
-        ),
-    ]

confirmation/migrations/0007_add_indexes.py

@@ -1,37 +0,0 @@
-# Generated by Django 2.2.10 on 2020-03-27 09:02
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('confirmation', '0006_realmcreationkey_presume_email_valid'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='confirmation',
-            name='confirmation_key',
-            field=models.CharField(db_index=True, max_length=40),
-        ),
-        migrations.AlterField(
-            model_name='confirmation',
-            name='date_sent',
-            field=models.DateTimeField(db_index=True),
-        ),
-        migrations.AlterField(
-            model_name='confirmation',
-            name='object_id',
-            field=models.PositiveIntegerField(db_index=True),
-        ),
-        migrations.AlterField(
-            model_name='realmcreationkey',
-            name='creation_key',
-            field=models.CharField(db_index=True, max_length=40, verbose_name='activation key'),
-        ),
-        migrations.AlterUniqueTogether(
-            name='confirmation',
-            unique_together={('type', 'confirmation_key')},
-        ),
-    ]

confirmation/models.py

@@ -1,56 +1,59 @@
+# -*- coding: utf-8 -*-
+
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
 __revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $'
-import datetime
-import string
-from random import SystemRandom
-from typing import Mapping, Optional, Union
-from urllib.parse import urljoin
 
-from django.conf import settings
-from django.contrib.contenttypes.fields import GenericForeignKey
-from django.contrib.contenttypes.models import ContentType
+import datetime
+
 from django.db import models
-from django.db.models import CASCADE
+from django.core.urlresolvers import reverse
+from django.conf import settings
+from django.contrib.sites.models import Site
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericForeignKey
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
-from django.urls import reverse
 from django.utils.timezone import now as timezone_now
 
-from zerver.models import EmailChangeStatus, MultiuseInvite, PreregistrationUser, Realm, UserProfile
-
+from zerver.lib.send_email import send_email
+from zerver.lib.utils import generate_random_token
+from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite
+from random import SystemRandom
+import string
+from typing import Any, Dict, Optional, Text, Union
 
 class ConfirmationKeyException(Exception):
     WRONG_LENGTH = 1
     EXPIRED = 2
     DOES_NOT_EXIST = 3
 
-    def __init__(self, error_type: int) -> None:
-        super().__init__()
+    def __init__(self, error_type):
+        # type: (int) -> None
+        super(ConfirmationKeyException, self).__init__()
         self.error_type = error_type
 
-def render_confirmation_key_error(request: HttpRequest, exception: ConfirmationKeyException) -> HttpResponse:
+def render_confirmation_key_error(request, exception):
+    # type: (HttpRequest, ConfirmationKeyException) -> HttpResponse
     if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
         return render(request, 'confirmation/link_malformed.html')
     if exception.error_type == ConfirmationKeyException.EXPIRED:
         return render(request, 'confirmation/link_expired.html')
     return render(request, 'confirmation/link_does_not_exist.html')
 
-def generate_key() -> str:
+def generate_key():
+    # type: () -> str
     generator = SystemRandom()
     # 24 characters * 5 bits of entropy/character = 120 bits of entropy
     return ''.join(generator.choice(string.ascii_lowercase + string.digits) for _ in range(24))
 
-ConfirmationObjT = Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
-def get_object_from_key(confirmation_key: str,
-                        confirmation_type: int,
-                        activate_object: bool=True) -> ConfirmationObjT:
+def get_object_from_key(confirmation_key):
+    # type: (str) -> Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
     # Confirmation keys used to be 40 characters
     if len(confirmation_key) not in (24, 40):
         raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
     try:
-        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key,
-                                                type=confirmation_type)
+        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key)
     except Confirmation.DoesNotExist:
         raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)
 
@@ -59,42 +62,32 @@ def get_object_from_key(confirmation_key: str,
         raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)
 
     obj = confirmation.content_object
-    if activate_object and hasattr(obj, "status"):
+    if hasattr(obj, "status"):
         obj.status = getattr(settings, 'STATUS_ACTIVE', 1)
         obj.save(update_fields=['status'])
     return obj
 
-def create_confirmation_link(obj: ContentType,
-                             confirmation_type: int,
-                             url_args: Mapping[str, str] = {}) -> str:
+def create_confirmation_link(obj, host, confirmation_type, url_args=None):
+    # type: (Union[ContentType, int], str, int, Optional[Dict[str, str]]) -> str
     key = generate_key()
-    realm = None
-    if hasattr(obj, 'realm'):
-        realm = obj.realm
-    elif isinstance(obj, Realm):
-        realm = obj
-
     Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
-                                realm=realm, type=confirmation_type)
-    return confirmation_url(key, realm, confirmation_type, url_args)
+                                type=confirmation_type)
+    return confirmation_url(key, host, confirmation_type, url_args)
 
-def confirmation_url(confirmation_key: str, realm: Optional[Realm],
-                     confirmation_type: int,
-                     url_args: Mapping[str, str] = {}) -> str:
-    url_args = dict(url_args)
+def confirmation_url(confirmation_key, host, confirmation_type, url_args=None):
+    # type: (str, str, int, Optional[Dict[str, str]]) -> str
+    if url_args is None:
+        url_args = {}
     url_args['confirmation_key'] = confirmation_key
-    return urljoin(
-        settings.ROOT_DOMAIN_URI if realm is None else realm.uri,
-        reverse(_properties[confirmation_type].url_name, kwargs=url_args),
-    )
+    return '%s%s%s' % (settings.EXTERNAL_URI_SCHEME, host,
+                       reverse(_properties[confirmation_type].url_name, kwargs=url_args))
 
 class Confirmation(models.Model):
-    content_type = models.ForeignKey(ContentType, on_delete=CASCADE)
-    object_id: int = models.PositiveIntegerField(db_index=True)
+    content_type = models.ForeignKey(ContentType)
+    object_id = models.PositiveIntegerField()  # type: int
     content_object = GenericForeignKey('content_type', 'object_id')
-    date_sent: datetime.datetime = models.DateTimeField(db_index=True)
-    confirmation_key: str = models.CharField(max_length=40, db_index=True)
-    realm: Optional[Realm] = models.ForeignKey(Realm, null=True, on_delete=CASCADE)
+    date_sent = models.DateTimeField()  # type: datetime.datetime
+    confirmation_key = models.CharField(max_length=40)  # type: str
 
     # The following list is the set of valid types
     USER_REGISTRATION = 1
@@ -103,83 +96,51 @@ class Confirmation(models.Model):
     UNSUBSCRIBE = 4
     SERVER_REGISTRATION = 5
     MULTIUSE_INVITE = 6
-    REALM_CREATION = 7
-    REALM_REACTIVATION = 8
-    type: int = models.PositiveSmallIntegerField()
+    type = models.PositiveSmallIntegerField()  # type: int
 
-    def __str__(self) -> str:
-        return f'<Confirmation: {self.content_object}>'
+    def __unicode__(self):
+        # type: () -> Text
+        return '<Confirmation: %s>' % (self.content_object,)
 
-    class Meta:
-        unique_together = ("type", "confirmation_key")
-
-class ConfirmationType:
-    def __init__(self, url_name: str,
-                 validity_in_days: int=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS) -> None:
+class ConfirmationType(object):
+    def __init__(self, url_name, validity_in_days=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS):
+        # type: (str, int) -> None
         self.url_name = url_name
         self.validity_in_days = validity_in_days
 
 _properties = {
-    Confirmation.USER_REGISTRATION: ConfirmationType('check_prereg_key_and_redirect'),
-    Confirmation.INVITATION: ConfirmationType('check_prereg_key_and_redirect',
+    Confirmation.USER_REGISTRATION: ConfirmationType('confirmation.views.confirm'),
+    Confirmation.INVITATION: ConfirmationType('confirmation.views.confirm',
                                               validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
     Confirmation.EMAIL_CHANGE: ConfirmationType('zerver.views.user_settings.confirm_email_change'),
     Confirmation.UNSUBSCRIBE: ConfirmationType('zerver.views.unsubscribe.email_unsubscribe',
                                                validity_in_days=1000000),  # should never expire
-    Confirmation.MULTIUSE_INVITE: ConfirmationType(
-        'zerver.views.registration.accounts_home_from_multiuse_invite',
-        validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
-    Confirmation.REALM_CREATION: ConfirmationType('check_prereg_key_and_redirect'),
-    Confirmation.REALM_REACTIVATION: ConfirmationType('zerver.views.realm.realm_reactivation'),
+    Confirmation.MULTIUSE_INVITE: ConfirmationType('zerver.views.registration.accounts_home_from_multiuse_invite',
+                                                   validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS)
 }
 
-def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> str:
-    """
-    Generate a unique link that a logged-out user can visit to unsubscribe from
-    Zulip e-mails without having to first log in.
-    """
-    return create_confirmation_link(user_profile,
-                                    Confirmation.UNSUBSCRIBE,
-                                    url_args = {'email_type': email_type})
+# Conirmation pathways for which there is no content_object that we need to
+# keep track of.
 
-# Functions related to links generated by the generate_realm_creation_link.py
-# management command.
-# Note that being validated here will just allow the user to access the create_realm
-# form, where they will enter their email and go through the regular
-# Confirmation.REALM_CREATION pathway.
-# Arguably RealmCreationKey should just be another ConfirmationObjT and we should
-# add another Confirmation.type for this; it's this way for historical reasons.
+def check_key_is_valid(creation_key):
+    # type: (Text) -> bool
+    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
+        return False
+    days_sofar = (timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created).days
+    # Realm creation link expires after settings.REALM_CREATION_LINK_VALIDITY_DAYS
+    if days_sofar <= settings.REALM_CREATION_LINK_VALIDITY_DAYS:
+        return True
+    return False
 
-def validate_key(creation_key: Optional[str]) -> Optional['RealmCreationKey']:
-    """Get the record for this key, raising InvalidCreationKey if non-None but invalid."""
-    if creation_key is None:
-        return None
-    try:
-        key_record = RealmCreationKey.objects.get(creation_key=creation_key)
-    except RealmCreationKey.DoesNotExist:
-        raise RealmCreationKey.Invalid()
-    time_elapsed = timezone_now() - key_record.date_created
-    if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600:
-        raise RealmCreationKey.Invalid()
-    return key_record
-
-def generate_realm_creation_url(by_admin: bool=False) -> str:
+def generate_realm_creation_url():
+    # type: () -> Text
     key = generate_key()
-    RealmCreationKey.objects.create(creation_key=key,
-                                    date_created=timezone_now(),
-                                    presume_email_valid=by_admin)
-    return urljoin(
-        settings.ROOT_DOMAIN_URI,
-        reverse('zerver.views.create_realm', kwargs={'creation_key': key}),
-    )
+    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
+    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
+                        settings.EXTERNAL_HOST,
+                        reverse('zerver.views.create_realm',
+                                kwargs={'creation_key': key}))
 
 class RealmCreationKey(models.Model):
-    creation_key = models.CharField('activation key', db_index=True, max_length=40)
+    creation_key = models.CharField('activation key', max_length=40)
     date_created = models.DateTimeField('created', default=timezone_now)
-
-    # True just if we should presume the email address the user enters
-    # is theirs, and skip sending mail to it to confirm that.
-    presume_email_valid: bool = models.BooleanField(default=False)
-
-    class Invalid(Exception):
-        pass

confirmation/settings.py

@@ -1,6 +1,9 @@
+# -*- coding: utf-8 -*-
+
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
+from typing import Any, Dict
+
 __revision__ = '$Id: settings.py 12 2008-11-23 19:38:52Z jarek.zgoda $'
 
 STATUS_ACTIVE = 1
-STATUS_REVOKED = 2

confirmation/views.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
+
+__revision__ = '$Id: views.py 21 2008-12-05 09:21:03Z jarek.zgoda $'
+
+
+from django.shortcuts import render
+from django.template import RequestContext
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+
+from confirmation.models import Confirmation, get_object_from_key, ConfirmationKeyException, \
+    render_confirmation_key_error
+from zerver.models import PreregistrationUser
+
+from typing import Any, Dict
+
+# This is currently only used for confirming PreregistrationUser.
+# Do not add other confirmation paths here.
+def confirm(request, confirmation_key):
+    # type: (HttpRequest, str) -> HttpResponse
+    try:
+        get_object_from_key(confirmation_key)
+    except ConfirmationKeyException as exception:
+        return render_confirmation_key_error(request, exception)
+
+    return render(request, 'confirmation/confirm_preregistrationuser.html',
+                  context={
+                      'key': confirmation_key,
+                      'full_name': request.GET.get("full_name", None)})

corporate/lib/stripe.py

@@ -1,610 +0,0 @@
-import logging
-import math
-import os
-from datetime import datetime, timedelta
-from decimal import Decimal
-from functools import wraps
-from typing import Callable, Dict, Optional, Tuple, TypeVar, cast
-
-import stripe
-import ujson
-from django.conf import settings
-from django.core.signing import Signer
-from django.db import transaction
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import ugettext as _
-
-from corporate.models import (
-    Customer,
-    CustomerPlan,
-    LicenseLedger,
-    get_current_plan_by_customer,
-    get_current_plan_by_realm,
-    get_customer_by_realm,
-)
-from zerver.lib.logging_util import log_to_file
-from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime
-from zerver.lib.utils import generate_random_token
-from zerver.models import Realm, RealmAuditLog, UserProfile
-from zproject.config import get_secret
-
-STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key')
-stripe.api_key = get_secret('stripe_secret_key')
-
-BILLING_LOG_PATH = os.path.join('/var/log/zulip'
-                                if not settings.DEVELOPMENT
-                                else settings.DEVELOPMENT_LOG_DIRECTORY,
-                                'billing.log')
-billing_logger = logging.getLogger('corporate.stripe')
-log_to_file(billing_logger, BILLING_LOG_PATH)
-log_to_file(logging.getLogger('stripe'), BILLING_LOG_PATH)
-
-CallableT = TypeVar('CallableT', bound=Callable[..., object])
-
-MIN_INVOICED_LICENSES = 30
-MAX_INVOICED_LICENSES = 1000
-DEFAULT_INVOICE_DAYS_UNTIL_DUE = 30
-
-def get_latest_seat_count(realm: Realm) -> int:
-    non_guests = UserProfile.objects.filter(
-        realm=realm, is_active=True, is_bot=False).exclude(role=UserProfile.ROLE_GUEST).count()
-    guests = UserProfile.objects.filter(
-        realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST).count()
-    return max(non_guests, math.ceil(guests / 5))
-
-def sign_string(string: str) -> Tuple[str, str]:
-    salt = generate_random_token(64)
-    signer = Signer(salt=salt)
-    return signer.sign(string), salt
-
-def unsign_string(signed_string: str, salt: str) -> str:
-    signer = Signer(salt=salt)
-    return signer.unsign(signed_string)
-
-# Be extremely careful changing this function. Historical billing periods
-# are not stored anywhere, and are just computed on the fly using this
-# function. Any change you make here should return the same value (or be
-# within a few seconds) for basically any value from when the billing system
-# went online to within a year from now.
-def add_months(dt: datetime, months: int) -> datetime:
-    assert(months >= 0)
-    # It's fine that the max day in Feb is 28 for leap years.
-    MAX_DAY_FOR_MONTH = {1: 31, 2: 28, 3: 31, 4: 30, 5: 31, 6: 30,
-                         7: 31, 8: 31, 9: 30, 10: 31, 11: 30, 12: 31}
-    year = dt.year
-    month = dt.month + months
-    while month > 12:
-        year += 1
-        month -= 12
-    day = min(dt.day, MAX_DAY_FOR_MONTH[month])
-    # datetimes don't support leap seconds, so don't need to worry about those
-    return dt.replace(year=year, month=month, day=day)
-
-def next_month(billing_cycle_anchor: datetime, dt: datetime) -> datetime:
-    estimated_months = round((dt - billing_cycle_anchor).days * 12. / 365)
-    for months in range(max(estimated_months - 1, 0), estimated_months + 2):
-        proposed_next_month = add_months(billing_cycle_anchor, months)
-        if 20 < (proposed_next_month - dt).days < 40:
-            return proposed_next_month
-    raise AssertionError('Something wrong in next_month calculation with '
-                         f'billing_cycle_anchor: {billing_cycle_anchor}, dt: {dt}')
-
-def start_of_next_billing_cycle(plan: CustomerPlan, event_time: datetime) -> datetime:
-    if plan.status == CustomerPlan.FREE_TRIAL:
-        assert(plan.next_invoice_date is not None)  # for mypy
-        return plan.next_invoice_date
-
-    months_per_period = {
-        CustomerPlan.ANNUAL: 12,
-        CustomerPlan.MONTHLY: 1,
-    }[plan.billing_schedule]
-    periods = 1
-    dt = plan.billing_cycle_anchor
-    while dt <= event_time:
-        dt = add_months(plan.billing_cycle_anchor, months_per_period * periods)
-        periods += 1
-    return dt
-
-def next_invoice_date(plan: CustomerPlan) -> Optional[datetime]:
-    if plan.status == CustomerPlan.ENDED:
-        return None
-    assert(plan.next_invoice_date is not None)  # for mypy
-    months_per_period = {
-        CustomerPlan.ANNUAL: 12,
-        CustomerPlan.MONTHLY: 1,
-    }[plan.billing_schedule]
-    if plan.automanage_licenses:
-        months_per_period = 1
-    periods = 1
-    dt = plan.billing_cycle_anchor
-    while dt <= plan.next_invoice_date:
-        dt = add_months(plan.billing_cycle_anchor, months_per_period * periods)
-        periods += 1
-    return dt
-
-def renewal_amount(plan: CustomerPlan, event_time: datetime) -> int:  # nocoverage: TODO
-    if plan.fixed_price is not None:
-        return plan.fixed_price
-    new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, event_time)
-    if last_ledger_entry is None:
-        return 0
-    if last_ledger_entry.licenses_at_next_renewal is None:
-        return 0
-    if new_plan is not None:
-        plan = new_plan
-    assert(plan.price_per_license is not None)  # for mypy
-    return plan.price_per_license * last_ledger_entry.licenses_at_next_renewal
-
-def get_idempotency_key(ledger_entry: LicenseLedger) -> Optional[str]:
-    if settings.TEST_SUITE:
-        return None
-    return f'ledger_entry:{ledger_entry.id}'  # nocoverage
-
-class BillingError(Exception):
-    # error messages
-    CONTACT_SUPPORT = _("Something went wrong. Please contact {email}.").format(
-        email=settings.ZULIP_ADMINISTRATOR,
-    )
-    TRY_RELOADING = _("Something went wrong. Please reload the page.")
-
-    # description is used only for tests
-    def __init__(self, description: str, message: str=CONTACT_SUPPORT) -> None:
-        self.description = description
-        self.message = message
-
-class StripeCardError(BillingError):
-    pass
-
-class StripeConnectionError(BillingError):
-    pass
-
-def catch_stripe_errors(func: CallableT) -> CallableT:
-    @wraps(func)
-    def wrapped(*args: object, **kwargs: object) -> object:
-        if settings.DEVELOPMENT and not settings.TEST_SUITE:  # nocoverage
-            if STRIPE_PUBLISHABLE_KEY is None:
-                raise BillingError('missing stripe config', "Missing Stripe config. "
-                                   "See https://zulip.readthedocs.io/en/latest/subsystems/billing.html.")
-        try:
-            return func(*args, **kwargs)
-        # See https://stripe.com/docs/api/python#error_handling, though
-        # https://stripe.com/docs/api/ruby#error_handling suggests there are additional fields, and
-        # https://stripe.com/docs/error-codes gives a more detailed set of error codes
-        except stripe.error.StripeError as e:
-            err = e.json_body.get('error', {})
-            billing_logger.error(
-                "Stripe error: %s %s %s %s",
-                e.http_status, err.get('type'), err.get('code'), err.get('param'),
-            )
-            if isinstance(e, stripe.error.CardError):
-                # TODO: Look into i18n for this
-                raise StripeCardError('card error', err.get('message'))
-            if isinstance(e, (stripe.error.RateLimitError, stripe.error.APIConnectionError)):  # nocoverage TODO
-                raise StripeConnectionError(
-                    'stripe connection error',
-                    _("Something went wrong. Please wait a few seconds and try again."))
-            raise BillingError('other stripe error', BillingError.CONTACT_SUPPORT)
-    return cast(CallableT, wrapped)
-
-@catch_stripe_errors
-def stripe_get_customer(stripe_customer_id: str) -> stripe.Customer:
-    return stripe.Customer.retrieve(stripe_customer_id, expand=["default_source"])
-
-@catch_stripe_errors
-def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str]=None) -> Customer:
-    realm = user.realm
-    # We could do a better job of handling race conditions here, but if two
-    # people from a realm try to upgrade at exactly the same time, the main
-    # bad thing that will happen is that we will create an extra stripe
-    # customer that we can delete or ignore.
-    stripe_customer = stripe.Customer.create(
-        description=f"{realm.string_id} ({realm.name})",
-        email=user.delivery_email,
-        metadata={'realm_id': realm.id, 'realm_str': realm.string_id},
-        source=stripe_token)
-    event_time = timestamp_to_datetime(stripe_customer.created)
-    with transaction.atomic():
-        RealmAuditLog.objects.create(
-            realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CUSTOMER_CREATED,
-            event_time=event_time)
-        if stripe_token is not None:
-            RealmAuditLog.objects.create(
-                realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CARD_CHANGED,
-                event_time=event_time)
-        customer, created = Customer.objects.update_or_create(realm=realm, defaults={
-            'stripe_customer_id': stripe_customer.id})
-        user.is_billing_admin = True
-        user.save(update_fields=["is_billing_admin"])
-    return customer
-
-@catch_stripe_errors
-def do_replace_payment_source(user: UserProfile, stripe_token: str,
-                              pay_invoices: bool=False) -> stripe.Customer:
-    customer = get_customer_by_realm(user.realm)
-    assert(customer is not None)  # for mypy
-
-    stripe_customer = stripe_get_customer(customer.stripe_customer_id)
-    stripe_customer.source = stripe_token
-    # Deletes existing card: https://stripe.com/docs/api#update_customer-source
-    updated_stripe_customer = stripe.Customer.save(stripe_customer)
-    RealmAuditLog.objects.create(
-        realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CARD_CHANGED,
-        event_time=timezone_now())
-    if pay_invoices:
-        for stripe_invoice in stripe.Invoice.list(
-                billing='charge_automatically', customer=stripe_customer.id, status='open'):
-            # The user will get either a receipt or a "failed payment" email, but the in-app
-            # messaging could be clearer here (e.g. it could explicitly tell the user that there
-            # were payment(s) and that they succeeded or failed).
-            # Worth fixing if we notice that a lot of cards end up failing at this step.
-            stripe.Invoice.pay(stripe_invoice)
-    return updated_stripe_customer
-
-# event_time should roughly be timezone_now(). Not designed to handle
-# event_times in the past or future
-@transaction.atomic
-def make_end_of_cycle_updates_if_needed(plan: CustomerPlan,
-                                        event_time: datetime) -> Tuple[Optional[CustomerPlan], Optional[LicenseLedger]]:
-    last_ledger_entry = LicenseLedger.objects.filter(plan=plan).order_by('-id').first()
-    last_renewal = LicenseLedger.objects.filter(plan=plan, is_renewal=True) \
-                                        .order_by('-id').first().event_time
-    next_billing_cycle = start_of_next_billing_cycle(plan, last_renewal)
-    if next_billing_cycle <= event_time:
-        if plan.status == CustomerPlan.ACTIVE:
-            return None, LicenseLedger.objects.create(
-                plan=plan, is_renewal=True, event_time=next_billing_cycle,
-                licenses=last_ledger_entry.licenses_at_next_renewal,
-                licenses_at_next_renewal=last_ledger_entry.licenses_at_next_renewal)
-        if plan.status == CustomerPlan.FREE_TRIAL:
-            plan.invoiced_through = last_ledger_entry
-            assert(plan.next_invoice_date is not None)
-            plan.billing_cycle_anchor = plan.next_invoice_date.replace(microsecond=0)
-            plan.status = CustomerPlan.ACTIVE
-            plan.save(update_fields=["invoiced_through", "billing_cycle_anchor", "status"])
-            return None, LicenseLedger.objects.create(
-                plan=plan, is_renewal=True, event_time=next_billing_cycle,
-                licenses=last_ledger_entry.licenses_at_next_renewal,
-                licenses_at_next_renewal=last_ledger_entry.licenses_at_next_renewal)
-
-        if plan.status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE:
-            if plan.fixed_price is not None:  # nocoverage
-                raise NotImplementedError("Can't switch fixed priced monthly plan to annual.")
-
-            plan.status = CustomerPlan.ENDED
-            plan.save(update_fields=["status"])
-
-            discount = plan.customer.default_discount or plan.discount
-            _, _, _, price_per_license = compute_plan_parameters(
-                automanage_licenses=plan.automanage_licenses, billing_schedule=CustomerPlan.ANNUAL,
-                discount=plan.discount
-            )
-
-            new_plan = CustomerPlan.objects.create(
-                customer=plan.customer, billing_schedule=CustomerPlan.ANNUAL, automanage_licenses=plan.automanage_licenses,
-                charge_automatically=plan.charge_automatically, price_per_license=price_per_license,
-                discount=discount, billing_cycle_anchor=next_billing_cycle,
-                tier=plan.tier, status=CustomerPlan.ACTIVE, next_invoice_date=next_billing_cycle,
-                invoiced_through=None, invoicing_status=CustomerPlan.INITIAL_INVOICE_TO_BE_SENT,
-            )
-
-            new_plan_ledger_entry = LicenseLedger.objects.create(
-                plan=new_plan, is_renewal=True, event_time=next_billing_cycle,
-                licenses=last_ledger_entry.licenses_at_next_renewal,
-                licenses_at_next_renewal=last_ledger_entry.licenses_at_next_renewal
-            )
-
-            RealmAuditLog.objects.create(
-                realm=new_plan.customer.realm, event_time=event_time,
-                event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN,
-                extra_data=ujson.dumps({
-                    "monthly_plan_id": plan.id,
-                    "annual_plan_id": new_plan.id,
-                })
-            )
-            return new_plan, new_plan_ledger_entry
-
-        if plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE:
-            process_downgrade(plan)
-        return None, None
-    return None, last_ledger_entry
-
-# Returns Customer instead of stripe_customer so that we don't make a Stripe
-# API call if there's nothing to update
-def update_or_create_stripe_customer(user: UserProfile, stripe_token: Optional[str]=None) -> Customer:
-    realm = user.realm
-    customer = get_customer_by_realm(realm)
-    if customer is None or customer.stripe_customer_id is None:
-        return do_create_stripe_customer(user, stripe_token=stripe_token)
-    if stripe_token is not None:
-        do_replace_payment_source(user, stripe_token)
-    return customer
-
-def compute_plan_parameters(
-        automanage_licenses: bool, billing_schedule: int,
-        discount: Optional[Decimal],
-        free_trial: bool=False) -> Tuple[datetime, datetime, datetime, int]:
-    # Everything in Stripe is stored as timestamps with 1 second resolution,
-    # so standardize on 1 second resolution.
-    # TODO talk about leapseconds?
-    billing_cycle_anchor = timezone_now().replace(microsecond=0)
-    if billing_schedule == CustomerPlan.ANNUAL:
-        # TODO use variables to account for Zulip Plus
-        price_per_license = 8000
-        period_end = add_months(billing_cycle_anchor, 12)
-    elif billing_schedule == CustomerPlan.MONTHLY:
-        price_per_license = 800
-        period_end = add_months(billing_cycle_anchor, 1)
-    else:
-        raise AssertionError(f'Unknown billing_schedule: {billing_schedule}')
-    if discount is not None:
-        # There are no fractional cents in Stripe, so round down to nearest integer.
-        price_per_license = int(float(price_per_license * (1 - discount / 100)) + .00001)
-    next_invoice_date = period_end
-    if automanage_licenses:
-        next_invoice_date = add_months(billing_cycle_anchor, 1)
-    if free_trial:
-        period_end = billing_cycle_anchor + timedelta(days=settings.FREE_TRIAL_DAYS)
-        next_invoice_date = period_end
-    return billing_cycle_anchor, next_invoice_date, period_end, price_per_license
-
-# Only used for cloud signups
-@catch_stripe_errors
-def process_initial_upgrade(user: UserProfile, licenses: int, automanage_licenses: bool,
-                            billing_schedule: int, stripe_token: Optional[str]) -> None:
-    realm = user.realm
-    customer = update_or_create_stripe_customer(user, stripe_token=stripe_token)
-    charge_automatically = stripe_token is not None
-    free_trial = settings.FREE_TRIAL_DAYS not in (None, 0)
-
-    if get_current_plan_by_customer(customer) is not None:
-        # Unlikely race condition from two people upgrading (clicking "Make payment")
-        # at exactly the same time. Doesn't fully resolve the race condition, but having
-        # a check here reduces the likelihood.
-        billing_logger.warning(
-            "Customer %s trying to upgrade, but has an active subscription", customer,
-        )
-        raise BillingError('subscribing with existing subscription', BillingError.TRY_RELOADING)
-
-    billing_cycle_anchor, next_invoice_date, period_end, price_per_license = compute_plan_parameters(
-        automanage_licenses, billing_schedule, customer.default_discount, free_trial)
-    # The main design constraint in this function is that if you upgrade with a credit card, and the
-    # charge fails, everything should be rolled back as if nothing had happened. This is because we
-    # expect frequent card failures on initial signup.
-    # Hence, if we're going to charge a card, do it at the beginning, even if we later may have to
-    # adjust the number of licenses.
-    if charge_automatically:
-        if not free_trial:
-            stripe_charge = stripe.Charge.create(
-                amount=price_per_license * licenses,
-                currency='usd',
-                customer=customer.stripe_customer_id,
-                description=f"Upgrade to Zulip Standard, ${price_per_license/100} x {licenses}",
-                receipt_email=user.delivery_email,
-                statement_descriptor='Zulip Standard')
-            # Not setting a period start and end, but maybe we should? Unclear what will make things
-            # most similar to the renewal case from an accounting perspective.
-            assert isinstance(stripe_charge.source, stripe.Card)
-            description = f"Payment (Card ending in {stripe_charge.source.last4})"
-            stripe.InvoiceItem.create(
-                amount=price_per_license * licenses * -1,
-                currency='usd',
-                customer=customer.stripe_customer_id,
-                description=description,
-                discountable=False)
-
-    # TODO: The correctness of this relies on user creation, deactivation, etc being
-    # in a transaction.atomic() with the relevant RealmAuditLog entries
-    with transaction.atomic():
-        # billed_licenses can greater than licenses if users are added between the start of
-        # this function (process_initial_upgrade) and now
-        billed_licenses = max(get_latest_seat_count(realm), licenses)
-        plan_params = {
-            'automanage_licenses': automanage_licenses,
-            'charge_automatically': charge_automatically,
-            'price_per_license': price_per_license,
-            'discount': customer.default_discount,
-            'billing_cycle_anchor': billing_cycle_anchor,
-            'billing_schedule': billing_schedule,
-            'tier': CustomerPlan.STANDARD}
-        if free_trial:
-            plan_params['status'] = CustomerPlan.FREE_TRIAL
-        plan = CustomerPlan.objects.create(
-            customer=customer,
-            next_invoice_date=next_invoice_date,
-            **plan_params)
-        ledger_entry = LicenseLedger.objects.create(
-            plan=plan,
-            is_renewal=True,
-            event_time=billing_cycle_anchor,
-            licenses=billed_licenses,
-            licenses_at_next_renewal=billed_licenses)
-        plan.invoiced_through = ledger_entry
-        plan.save(update_fields=['invoiced_through'])
-        RealmAuditLog.objects.create(
-            realm=realm, acting_user=user, event_time=billing_cycle_anchor,
-            event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED,
-            extra_data=ujson.dumps(plan_params))
-
-    if not free_trial:
-        stripe.InvoiceItem.create(
-            currency='usd',
-            customer=customer.stripe_customer_id,
-            description='Zulip Standard',
-            discountable=False,
-            period = {'start': datetime_to_timestamp(billing_cycle_anchor),
-                      'end': datetime_to_timestamp(period_end)},
-            quantity=billed_licenses,
-            unit_amount=price_per_license)
-
-        if charge_automatically:
-            billing_method = 'charge_automatically'
-            days_until_due = None
-        else:
-            billing_method = 'send_invoice'
-            days_until_due = DEFAULT_INVOICE_DAYS_UNTIL_DUE
-
-        stripe_invoice = stripe.Invoice.create(
-            auto_advance=True,
-            billing=billing_method,
-            customer=customer.stripe_customer_id,
-            days_until_due=days_until_due,
-            statement_descriptor='Zulip Standard')
-        stripe.Invoice.finalize_invoice(stripe_invoice)
-
-    from zerver.lib.actions import do_change_plan_type
-    do_change_plan_type(realm, Realm.STANDARD)
-
-def update_license_ledger_for_automanaged_plan(realm: Realm, plan: CustomerPlan,
-                                               event_time: datetime) -> None:
-    new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(plan, event_time)
-    if last_ledger_entry is None:
-        return
-    if new_plan is not None:
-        plan = new_plan
-    licenses_at_next_renewal = get_latest_seat_count(realm)
-    licenses = max(licenses_at_next_renewal, last_ledger_entry.licenses)
-
-    LicenseLedger.objects.create(
-        plan=plan, event_time=event_time, licenses=licenses,
-        licenses_at_next_renewal=licenses_at_next_renewal)
-
-def update_license_ledger_if_needed(realm: Realm, event_time: datetime) -> None:
-    plan = get_current_plan_by_realm(realm)
-    if plan is None:
-        return
-    if not plan.automanage_licenses:
-        return
-    update_license_ledger_for_automanaged_plan(realm, plan, event_time)
-
-def invoice_plan(plan: CustomerPlan, event_time: datetime) -> None:
-    if plan.invoicing_status == CustomerPlan.STARTED:
-        raise NotImplementedError('Plan with invoicing_status==STARTED needs manual resolution.')
-    make_end_of_cycle_updates_if_needed(plan, event_time)
-
-    if plan.invoicing_status == CustomerPlan.INITIAL_INVOICE_TO_BE_SENT:
-        invoiced_through_id = -1
-        licenses_base = None
-    else:
-        assert(plan.invoiced_through is not None)
-        licenses_base = plan.invoiced_through.licenses
-        invoiced_through_id = plan.invoiced_through.id
-
-    invoice_item_created = False
-    for ledger_entry in LicenseLedger.objects.filter(plan=plan, id__gt=invoiced_through_id,
-                                                     event_time__lte=event_time).order_by('id'):
-        price_args: Dict[str, int] = {}
-        if ledger_entry.is_renewal:
-            if plan.fixed_price is not None:
-                price_args = {'amount': plan.fixed_price}
-            else:
-                assert(plan.price_per_license is not None)  # needed for mypy
-                price_args = {'unit_amount': plan.price_per_license,
-                              'quantity': ledger_entry.licenses}
-            description = "Zulip Standard - renewal"
-        elif licenses_base is not None and ledger_entry.licenses != licenses_base:
-            assert(plan.price_per_license)
-            last_renewal = LicenseLedger.objects.filter(
-                plan=plan, is_renewal=True, event_time__lte=ledger_entry.event_time) \
-                .order_by('-id').first().event_time
-            period_end = start_of_next_billing_cycle(plan, ledger_entry.event_time)
-            proration_fraction = (period_end - ledger_entry.event_time) / (period_end - last_renewal)
-            price_args = {'unit_amount': int(plan.price_per_license * proration_fraction + .5),
-                          'quantity': ledger_entry.licenses - licenses_base}
-            description = "Additional license ({} - {})".format(
-                ledger_entry.event_time.strftime('%b %-d, %Y'), period_end.strftime('%b %-d, %Y'))
-
-        if price_args:
-            plan.invoiced_through = ledger_entry
-            plan.invoicing_status = CustomerPlan.STARTED
-            plan.save(update_fields=['invoicing_status', 'invoiced_through'])
-            stripe.InvoiceItem.create(
-                currency='usd',
-                customer=plan.customer.stripe_customer_id,
-                description=description,
-                discountable=False,
-                period = {'start': datetime_to_timestamp(ledger_entry.event_time),
-                          'end': datetime_to_timestamp(
-                              start_of_next_billing_cycle(plan, ledger_entry.event_time))},
-                idempotency_key=get_idempotency_key(ledger_entry),
-                **price_args)
-            invoice_item_created = True
-        plan.invoiced_through = ledger_entry
-        plan.invoicing_status = CustomerPlan.DONE
-        plan.save(update_fields=['invoicing_status', 'invoiced_through'])
-        licenses_base = ledger_entry.licenses
-
-    if invoice_item_created:
-        if plan.charge_automatically:
-            billing_method = 'charge_automatically'
-            days_until_due = None
-        else:
-            billing_method = 'send_invoice'
-            days_until_due = DEFAULT_INVOICE_DAYS_UNTIL_DUE
-        stripe_invoice = stripe.Invoice.create(
-            auto_advance=True,
-            billing=billing_method,
-            customer=plan.customer.stripe_customer_id,
-            days_until_due=days_until_due,
-            statement_descriptor='Zulip Standard')
-        stripe.Invoice.finalize_invoice(stripe_invoice)
-
-    plan.next_invoice_date = next_invoice_date(plan)
-    plan.save(update_fields=['next_invoice_date'])
-
-def invoice_plans_as_needed(event_time: datetime=timezone_now()) -> None:
-    for plan in CustomerPlan.objects.filter(next_invoice_date__lte=event_time):
-        invoice_plan(plan, event_time)
-
-def attach_discount_to_realm(realm: Realm, discount: Decimal) -> None:
-    Customer.objects.update_or_create(realm=realm, defaults={'default_discount': discount})
-
-def update_sponsorship_status(realm: Realm, sponsorship_pending: bool) -> None:
-    customer, _ = Customer.objects.get_or_create(realm=realm)
-    customer.sponsorship_pending = sponsorship_pending
-    customer.save(update_fields=["sponsorship_pending"])
-
-def get_discount_for_realm(realm: Realm) -> Optional[Decimal]:
-    customer = get_customer_by_realm(realm)
-    if customer is not None:
-        return customer.default_discount
-    return None
-
-def do_change_plan_status(plan: CustomerPlan, status: int) -> None:
-    plan.status = status
-    plan.save(update_fields=['status'])
-    billing_logger.info(
-        'Change plan status: Customer.id: %s, CustomerPlan.id: %s, status: %s',
-        plan.customer.id, plan.id, status,
-    )
-
-def process_downgrade(plan: CustomerPlan) -> None:
-    from zerver.lib.actions import do_change_plan_type
-    do_change_plan_type(plan.customer.realm, Realm.LIMITED)
-    plan.status = CustomerPlan.ENDED
-    plan.save(update_fields=['status'])
-
-def estimate_annual_recurring_revenue_by_realm() -> Dict[str, int]:  # nocoverage
-    annual_revenue = {}
-    for plan in CustomerPlan.objects.filter(
-            status=CustomerPlan.ACTIVE).select_related('customer__realm'):
-        # TODO: figure out what to do for plans that don't automatically
-        # renew, but which probably will renew
-        renewal_cents = renewal_amount(plan, timezone_now())
-        if plan.billing_schedule == CustomerPlan.MONTHLY:
-            renewal_cents *= 12
-        # TODO: Decimal stuff
-        annual_revenue[plan.customer.realm.string_id] = int(renewal_cents / 100)
-    return annual_revenue
-
-# During realm deactivation we instantly downgrade the plan to Limited.
-# Extra users added in the final month are not charged. Also used
-# for the cancelation of Free Trial.
-def downgrade_now(realm: Realm) -> None:
-    plan = get_current_plan_by_realm(realm)
-    if plan is None:
-        return
-
-    process_downgrade(plan)
-    plan.invoiced_through = LicenseLedger.objects.filter(plan=plan).order_by('id').last()
-    plan.next_invoice_date = next_invoice_date(plan)
-    plan.save(update_fields=["invoiced_through", "next_invoice_date"])

corporate/migrations/0001_initial.py

@@ -1,51 +0,0 @@
-# Generated by Django 1.11.14 on 2018-09-25 12:02
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        ('zerver', '0189_userprofile_add_some_emojisets'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='BillingProcessor',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('state', models.CharField(max_length=20)),
-                ('last_modified', models.DateTimeField(auto_now=True)),
-                ('log_row', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.RealmAuditLog')),
-                ('realm', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Coupon',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('percent_off', models.SmallIntegerField(unique=True)),
-                ('stripe_coupon_id', models.CharField(max_length=255, unique=True)),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Customer',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('stripe_customer_id', models.CharField(max_length=255, unique=True)),
-                ('has_billing_relationship', models.BooleanField(default=False)),
-                ('realm', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
-            ],
-        ),
-        migrations.CreateModel(
-            name='Plan',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('nickname', models.CharField(max_length=40, unique=True)),
-                ('stripe_plan_id', models.CharField(max_length=255, unique=True)),
-            ],
-        ),
-    ]

corporate/migrations/0002_customer_default_discount.py

@@ -1,18 +0,0 @@
-# Generated by Django 1.11.16 on 2018-12-12 20:19
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='customer',
-            name='default_discount',
-            field=models.DecimalField(decimal_places=4, max_digits=7, null=True),
-        ),
-    ]

corporate/migrations/0003_customerplan.py

@@ -1,33 +0,0 @@
-# Generated by Django 1.11.16 on 2018-12-22 21:05
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0002_customer_default_discount'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='CustomerPlan',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('licenses', models.IntegerField()),
-                ('automanage_licenses', models.BooleanField(default=False)),
-                ('charge_automatically', models.BooleanField(default=False)),
-                ('price_per_license', models.IntegerField(null=True)),
-                ('fixed_price', models.IntegerField(null=True)),
-                ('discount', models.DecimalField(decimal_places=4, max_digits=6, null=True)),
-                ('billing_cycle_anchor', models.DateTimeField()),
-                ('billing_schedule', models.SmallIntegerField()),
-                ('billed_through', models.DateTimeField()),
-                ('next_billing_date', models.DateTimeField(db_index=True)),
-                ('tier', models.SmallIntegerField()),
-                ('status', models.SmallIntegerField(default=1)),
-                ('customer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='corporate.Customer')),
-            ],
-        ),
-    ]

corporate/migrations/0004_licenseledger.py

@@ -1,25 +0,0 @@
-# Generated by Django 1.11.18 on 2019-01-19 05:01
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0003_customerplan'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='LicenseLedger',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('is_renewal', models.BooleanField(default=False)),
-                ('event_time', models.DateTimeField()),
-                ('licenses', models.IntegerField()),
-                ('licenses_at_next_renewal', models.IntegerField(null=True)),
-                ('plan', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='corporate.CustomerPlan')),
-            ],
-        ),
-    ]

corporate/migrations/0005_customerplan_invoicing.py

@@ -1,33 +0,0 @@
-# Generated by Django 1.11.18 on 2019-01-28 13:04
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0004_licenseledger'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='customerplan',
-            old_name='next_billing_date',
-            new_name='next_invoice_date',
-        ),
-        migrations.RemoveField(
-            model_name='customerplan',
-            name='billed_through',
-        ),
-        migrations.AddField(
-            model_name='customerplan',
-            name='invoiced_through',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='corporate.LicenseLedger'),
-        ),
-        migrations.AddField(
-            model_name='customerplan',
-            name='invoicing_status',
-            field=models.SmallIntegerField(default=1),
-        ),
-    ]

corporate/migrations/0006_nullable_stripe_customer_id.py

@@ -1,18 +0,0 @@
-# Generated by Django 1.11.18 on 2019-01-29 01:46
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0005_customerplan_invoicing'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customer',
-            name='stripe_customer_id',
-            field=models.CharField(max_length=255, null=True, unique=True),
-        ),
-    ]

corporate/migrations/0007_remove_deprecated_fields.py

@@ -1,38 +0,0 @@
-# Generated by Django 1.11.18 on 2019-01-31 22:16
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0006_nullable_stripe_customer_id'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='billingprocessor',
-            name='log_row',
-        ),
-        migrations.RemoveField(
-            model_name='billingprocessor',
-            name='realm',
-        ),
-        migrations.DeleteModel(
-            name='Coupon',
-        ),
-        migrations.DeleteModel(
-            name='Plan',
-        ),
-        migrations.RemoveField(
-            model_name='customer',
-            name='has_billing_relationship',
-        ),
-        migrations.RemoveField(
-            model_name='customerplan',
-            name='licenses',
-        ),
-        migrations.DeleteModel(
-            name='BillingProcessor',
-        ),
-    ]

corporate/migrations/0008_nullable_next_invoice_date.py

@@ -1,18 +0,0 @@
-# Generated by Django 1.11.20 on 2019-04-11 00:45
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0007_remove_deprecated_fields'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='customerplan',
-            name='next_invoice_date',
-            field=models.DateTimeField(db_index=True, null=True),
-        ),
-    ]

corporate/migrations/0009_customer_sponsorship_pending.py

@@ -1,18 +0,0 @@
-# Generated by Django 2.2.13 on 2020-06-09 12:09
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('corporate', '0008_nullable_next_invoice_date'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='customer',
-            name='sponsorship_pending',
-            field=models.BooleanField(default=False),
-        ),
-    ]

corporate/models.py

@@ -1,87 +0,0 @@
-import datetime
-from decimal import Decimal
-from typing import Optional
-
-from django.db import models
-from django.db.models import CASCADE
-
-from zerver.models import Realm
-
-
-class Customer(models.Model):
-    realm: Realm = models.OneToOneField(Realm, on_delete=CASCADE)
-    stripe_customer_id: str = models.CharField(max_length=255, null=True, unique=True)
-    sponsorship_pending: bool = models.BooleanField(default=False)
-    # A percentage, like 85.
-    default_discount: Optional[Decimal] = models.DecimalField(decimal_places=4, max_digits=7, null=True)
-
-    def __str__(self) -> str:
-        return f"<Customer {self.realm} {self.stripe_customer_id}>"
-
-def get_customer_by_realm(realm: Realm) -> Optional[Customer]:
-    return Customer.objects.filter(realm=realm).first()
-
-class CustomerPlan(models.Model):
-    customer: Customer = models.ForeignKey(Customer, on_delete=CASCADE)
-    automanage_licenses: bool = models.BooleanField(default=False)
-    charge_automatically: bool = models.BooleanField(default=False)
-
-    # Both of these are in cents. Exactly one of price_per_license or
-    # fixed_price should be set. fixed_price is only for manual deals, and
-    # can't be set via the self-serve billing system.
-    price_per_license: Optional[int] = models.IntegerField(null=True)
-    fixed_price: Optional[int] = models.IntegerField(null=True)
-
-    # Discount that was applied. For display purposes only.
-    discount: Optional[Decimal] = models.DecimalField(decimal_places=4, max_digits=6, null=True)
-
-    billing_cycle_anchor: datetime.datetime = models.DateTimeField()
-    ANNUAL = 1
-    MONTHLY = 2
-    billing_schedule: int = models.SmallIntegerField()
-
-    next_invoice_date: Optional[datetime.datetime] = models.DateTimeField(db_index=True, null=True)
-    invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(
-        'LicenseLedger', null=True, on_delete=CASCADE, related_name='+')
-    DONE = 1
-    STARTED = 2
-    INITIAL_INVOICE_TO_BE_SENT = 3
-    invoicing_status: int = models.SmallIntegerField(default=DONE)
-
-    STANDARD = 1
-    PLUS = 2  # not available through self-serve signup
-    ENTERPRISE = 10
-    tier: int = models.SmallIntegerField()
-
-    ACTIVE = 1
-    DOWNGRADE_AT_END_OF_CYCLE = 2
-    FREE_TRIAL = 3
-    SWITCH_TO_ANNUAL_AT_END_OF_CYCLE = 4
-    # "Live" plans should have a value < LIVE_STATUS_THRESHOLD.
-    # There should be at most one live plan per customer.
-    LIVE_STATUS_THRESHOLD = 10
-    ENDED = 11
-    NEVER_STARTED = 12
-    status: int = models.SmallIntegerField(default=ACTIVE)
-
-    # TODO maybe override setattr to ensure billing_cycle_anchor, etc are immutable
-
-def get_current_plan_by_customer(customer: Customer) -> Optional[CustomerPlan]:
-    return CustomerPlan.objects.filter(
-        customer=customer, status__lt=CustomerPlan.LIVE_STATUS_THRESHOLD).first()
-
-def get_current_plan_by_realm(realm: Realm) -> Optional[CustomerPlan]:
-    customer = get_customer_by_realm(realm)
-    if customer is None:
-        return None
-    return get_current_plan_by_customer(customer)
-
-class LicenseLedger(models.Model):
-    plan: CustomerPlan = models.ForeignKey(CustomerPlan, on_delete=CASCADE)
-    # Also True for the initial upgrade.
-    is_renewal: bool = models.BooleanField(default=False)
-    event_time: datetime.datetime = models.DateTimeField()
-    licenses: int = models.IntegerField()
-    # None means the plan does not automatically renew.
-    # This cannot be None if plan.automanage_licenses.
-    licenses_at_next_renewal: Optional[int] = models.IntegerField(null=True)

corporate/tests/stripe_fixtures/attach_discount_to_realm--Charge.create.1.json

@@ -1,112 +0,0 @@
-{
-  "amount": 7200,
-  "amount_refunded": 0,
-  "application": null,
-  "application_fee": null,
-  "application_fee_amount": null,
-  "balance_transaction": "txn_NORMALIZED00000000000001",
-  "billing_details": {
-    "address": {
-      "city": "Pacific",
-      "country": "United States",
-      "line1": "Under the sea,",
-      "line2": null,
-      "postal_code": "33333",
-      "state": null
-    },
-    "email": null,
-    "name": "Ada Starr",
-    "phone": null
-  },
-  "captured": true,
-  "created": 1000000000,
-  "currency": "usd",
-  "customer": "cus_NORMALIZED0001",
-  "description": "Upgrade to Zulip Standard, $12.0 x 6",
-  "destination": null,
-  "dispute": null,
-  "failure_code": null,
-  "failure_message": null,
-  "fraud_details": {},
-  "id": "ch_NORMALIZED00000000000001",
-  "invoice": null,
-  "livemode": false,
-  "metadata": {},
-  "object": "charge",
-  "on_behalf_of": null,
-  "order": null,
-  "outcome": {
-    "network_status": "approved_by_network",
-    "reason": null,
-    "risk_level": "normal",
-    "risk_score": 00,
-    "seller_message": "Payment complete.",
-    "type": "authorized"
-  },
-  "paid": true,
-  "payment_intent": null,
-  "payment_method": "card_NORMALIZED00000000000001",
-  "payment_method_details": {
-    "card": {
-      "brand": "visa",
-      "checks": {
-        "address_line1_check": "pass",
-        "address_postal_code_check": "pass",
-        "cvc_check": "pass"
-      },
-      "country": "US",
-      "exp_month": 3,
-      "exp_year": 2033,
-      "fingerprint": "NORMALIZED000001",
-      "funding": "credit",
-      "last4": "4242",
-      "three_d_secure": null,
-      "wallet": null
-    },
-    "type": "card"
-  },
-  "receipt_email": "hamlet@zulip.com",
-  "receipt_number": null,
-  "receipt_url": "https://pay.stripe.com/receipts/acct_NORMALIZED000001/ch_NORMALIZED00000000000001/rcpt_NORMALIZED000000000000000000001",
-  "refunded": false,
-  "refunds": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/charges/ch_NORMALIZED00000000000001/refunds"
-  },
-  "review": null,
-  "shipping": null,
-  "source": {
-    "address_city": "Pacific",
-    "address_country": "United States",
-    "address_line1": "Under the sea,",
-    "address_line1_check": "pass",
-    "address_line2": null,
-    "address_state": null,
-    "address_zip": "33333",
-    "address_zip_check": "pass",
-    "brand": "Visa",
-    "country": "US",
-    "customer": "cus_NORMALIZED0001",
-    "cvc_check": "pass",
-    "dynamic_last4": null,
-    "exp_month": 3,
-    "exp_year": 2033,
-    "fingerprint": "NORMALIZED000001",
-    "funding": "credit",
-    "id": "card_NORMALIZED00000000000001",
-    "last4": "4242",
-    "metadata": {},
-    "name": "Ada Starr",
-    "object": "card",
-    "tokenization_method": null
-  },
-  "source_transfer": null,
-  "statement_descriptor": "Zulip Standard",
-  "statement_descriptor_suffix": "Zulip Standard",
-  "status": "succeeded",
-  "transfer_data": null,
-  "transfer_group": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Charge.create.2.json

@@ -1,112 +0,0 @@
-{
-  "amount": 36000,
-  "amount_refunded": 0,
-  "application": null,
-  "application_fee": null,
-  "application_fee_amount": null,
-  "balance_transaction": "txn_NORMALIZED00000000000002",
-  "billing_details": {
-    "address": {
-      "city": "Pacific",
-      "country": "United States",
-      "line1": "Under the sea,",
-      "line2": null,
-      "postal_code": "33333",
-      "state": null
-    },
-    "email": null,
-    "name": "Ada Starr",
-    "phone": null
-  },
-  "captured": true,
-  "created": 1000000000,
-  "currency": "usd",
-  "customer": "cus_NORMALIZED0001",
-  "description": "Upgrade to Zulip Standard, $60.0 x 6",
-  "destination": null,
-  "dispute": null,
-  "failure_code": null,
-  "failure_message": null,
-  "fraud_details": {},
-  "id": "ch_NORMALIZED00000000000002",
-  "invoice": null,
-  "livemode": false,
-  "metadata": {},
-  "object": "charge",
-  "on_behalf_of": null,
-  "order": null,
-  "outcome": {
-    "network_status": "approved_by_network",
-    "reason": null,
-    "risk_level": "normal",
-    "risk_score": 00,
-    "seller_message": "Payment complete.",
-    "type": "authorized"
-  },
-  "paid": true,
-  "payment_intent": null,
-  "payment_method": "card_NORMALIZED00000000000002",
-  "payment_method_details": {
-    "card": {
-      "brand": "visa",
-      "checks": {
-        "address_line1_check": "pass",
-        "address_postal_code_check": "pass",
-        "cvc_check": "pass"
-      },
-      "country": "US",
-      "exp_month": 3,
-      "exp_year": 2033,
-      "fingerprint": "NORMALIZED000001",
-      "funding": "credit",
-      "last4": "4242",
-      "three_d_secure": null,
-      "wallet": null
-    },
-    "type": "card"
-  },
-  "receipt_email": "hamlet@zulip.com",
-  "receipt_number": null,
-  "receipt_url": "https://pay.stripe.com/receipts/acct_NORMALIZED000001/ch_NORMALIZED00000000000002/rcpt_NORMALIZED000000000000000000002",
-  "refunded": false,
-  "refunds": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/charges/ch_NORMALIZED00000000000002/refunds"
-  },
-  "review": null,
-  "shipping": null,
-  "source": {
-    "address_city": "Pacific",
-    "address_country": "United States",
-    "address_line1": "Under the sea,",
-    "address_line1_check": "pass",
-    "address_line2": null,
-    "address_state": null,
-    "address_zip": "33333",
-    "address_zip_check": "pass",
-    "brand": "Visa",
-    "country": "US",
-    "customer": "cus_NORMALIZED0001",
-    "cvc_check": "pass",
-    "dynamic_last4": null,
-    "exp_month": 3,
-    "exp_year": 2033,
-    "fingerprint": "NORMALIZED000001",
-    "funding": "credit",
-    "id": "card_NORMALIZED00000000000002",
-    "last4": "4242",
-    "metadata": {},
-    "name": "Ada Starr",
-    "object": "card",
-    "tokenization_method": null
-  },
-  "source_transfer": null,
-  "statement_descriptor": "Zulip Standard",
-  "statement_descriptor_suffix": "Zulip Standard",
-  "status": "succeeded",
-  "transfer_data": null,
-  "transfer_group": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Charge.list.1.json

@@ -1,113 +0,0 @@
-{
-  "data": [
-    {
-      "amount": 7200,
-      "amount_refunded": 0,
-      "application": null,
-      "application_fee": null,
-      "application_fee_amount": null,
-      "balance_transaction": "txn_NORMALIZED00000000000001",
-      "billing_details": {
-        "address": {
-          "city": "Pacific",
-          "country": "United States",
-          "line1": "Under the sea,",
-          "line2": null,
-          "postal_code": "33333",
-          "state": null
-        },
-        "email": null,
-        "name": "Ada Starr",
-        "phone": null
-      },
-      "captured": true,
-      "created": 1000000000,
-      "currency": "usd",
-      "customer": "cus_NORMALIZED0001",
-      "description": "Upgrade to Zulip Standard, $12.0 x 6",
-      "destination": null,
-      "dispute": null,
-      "failure_code": null,
-      "failure_message": null,
-      "fraud_details": {},
-      "id": "ch_NORMALIZED00000000000001",
-      "invoice": null,
-      "livemode": false,
-      "metadata": {},
-      "object": "charge",
-      "on_behalf_of": null,
-      "order": null,
-      "outcome": {
-        "network_status": "approved_by_network",
-        "reason": null,
-        "risk_level": "normal",
-        "risk_score": 00,
-        "seller_message": "Payment complete.",
-        "type": "authorized"
-      },
-      "paid": true,
-      "payment_intent": null,
-      "payment_method": "card_NORMALIZED00000000000001",
-      "payment_method_details": {
-        "card": {
-          "brand": "visa",
-          "checks": {
-            "address_line1_check": "pass",
-            "address_postal_code_check": "pass",
-            "cvc_check": "pass"
-          },
-          "country": "US",
-          "exp_month": 3,
-          "exp_year": 2033,
-          "fingerprint": "NORMALIZED000001",
-          "funding": "credit",
-          "last4": "4242",
-          "three_d_secure": null,
-          "wallet": null
-        },
-        "type": "card"
-      },
-      "receipt_email": "hamlet@zulip.com",
-      "receipt_number": null,
-      "receipt_url": "https://pay.stripe.com/receipts/acct_NORMALIZED000001/ch_NORMALIZED00000000000001/rcpt_NORMALIZED000000000000000000001",
-      "refunded": false,
-      "refunds": {},
-      "review": null,
-      "shipping": null,
-      "source": {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000001",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      },
-      "source_transfer": null,
-      "statement_descriptor": "Zulip Standard",
-      "statement_descriptor_suffix": "Zulip Standard",
-      "status": "succeeded",
-      "transfer_data": null,
-      "transfer_group": null
-    }
-  ],
-  "has_more": false,
-  "object": "list",
-  "url": "/v1/charges"
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Charge.list.2.json

@@ -1,219 +0,0 @@
-{
-  "data": [
-    {
-      "amount": 36000,
-      "amount_refunded": 0,
-      "application": null,
-      "application_fee": null,
-      "application_fee_amount": null,
-      "balance_transaction": "txn_NORMALIZED00000000000002",
-      "billing_details": {
-        "address": {
-          "city": "Pacific",
-          "country": "United States",
-          "line1": "Under the sea,",
-          "line2": null,
-          "postal_code": "33333",
-          "state": null
-        },
-        "email": null,
-        "name": "Ada Starr",
-        "phone": null
-      },
-      "captured": true,
-      "created": 1000000000,
-      "currency": "usd",
-      "customer": "cus_NORMALIZED0001",
-      "description": "Upgrade to Zulip Standard, $60.0 x 6",
-      "destination": null,
-      "dispute": null,
-      "failure_code": null,
-      "failure_message": null,
-      "fraud_details": {},
-      "id": "ch_NORMALIZED00000000000002",
-      "invoice": null,
-      "livemode": false,
-      "metadata": {},
-      "object": "charge",
-      "on_behalf_of": null,
-      "order": null,
-      "outcome": {
-        "network_status": "approved_by_network",
-        "reason": null,
-        "risk_level": "normal",
-        "risk_score": 00,
-        "seller_message": "Payment complete.",
-        "type": "authorized"
-      },
-      "paid": true,
-      "payment_intent": null,
-      "payment_method": "card_NORMALIZED00000000000002",
-      "payment_method_details": {
-        "card": {
-          "brand": "visa",
-          "checks": {
-            "address_line1_check": "pass",
-            "address_postal_code_check": "pass",
-            "cvc_check": "pass"
-          },
-          "country": "US",
-          "exp_month": 3,
-          "exp_year": 2033,
-          "fingerprint": "NORMALIZED000001",
-          "funding": "credit",
-          "last4": "4242",
-          "three_d_secure": null,
-          "wallet": null
-        },
-        "type": "card"
-      },
-      "receipt_email": "hamlet@zulip.com",
-      "receipt_number": null,
-      "receipt_url": "https://pay.stripe.com/receipts/acct_NORMALIZED000001/ch_NORMALIZED00000000000002/rcpt_NORMALIZED000000000000000000002",
-      "refunded": false,
-      "refunds": {},
-      "review": null,
-      "shipping": null,
-      "source": {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000002",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      },
-      "source_transfer": null,
-      "statement_descriptor": "Zulip Standard",
-      "statement_descriptor_suffix": "Zulip Standard",
-      "status": "succeeded",
-      "transfer_data": null,
-      "transfer_group": null
-    },
-    {
-      "amount": 7200,
-      "amount_refunded": 0,
-      "application": null,
-      "application_fee": null,
-      "application_fee_amount": null,
-      "balance_transaction": "txn_NORMALIZED00000000000001",
-      "billing_details": {
-        "address": {
-          "city": "Pacific",
-          "country": "United States",
-          "line1": "Under the sea,",
-          "line2": null,
-          "postal_code": "33333",
-          "state": null
-        },
-        "email": null,
-        "name": "Ada Starr",
-        "phone": null
-      },
-      "captured": true,
-      "created": 1000000000,
-      "currency": "usd",
-      "customer": "cus_NORMALIZED0001",
-      "description": "Upgrade to Zulip Standard, $12.0 x 6",
-      "destination": null,
-      "dispute": null,
-      "failure_code": null,
-      "failure_message": null,
-      "fraud_details": {},
-      "id": "ch_NORMALIZED00000000000001",
-      "invoice": null,
-      "livemode": false,
-      "metadata": {},
-      "object": "charge",
-      "on_behalf_of": null,
-      "order": null,
-      "outcome": {
-        "network_status": "approved_by_network",
-        "reason": null,
-        "risk_level": "normal",
-        "risk_score": 00,
-        "seller_message": "Payment complete.",
-        "type": "authorized"
-      },
-      "paid": true,
-      "payment_intent": null,
-      "payment_method": "card_NORMALIZED00000000000001",
-      "payment_method_details": {
-        "card": {
-          "brand": "visa",
-          "checks": {
-            "address_line1_check": "pass",
-            "address_postal_code_check": "pass",
-            "cvc_check": "pass"
-          },
-          "country": "US",
-          "exp_month": 3,
-          "exp_year": 2033,
-          "fingerprint": "NORMALIZED000001",
-          "funding": "credit",
-          "last4": "4242",
-          "three_d_secure": null,
-          "wallet": null
-        },
-        "type": "card"
-      },
-      "receipt_email": "hamlet@zulip.com",
-      "receipt_number": null,
-      "receipt_url": "https://pay.stripe.com/receipts/acct_NORMALIZED000001/ch_NORMALIZED00000000000001/rcpt_NORMALIZED000000000000000000001",
-      "refunded": false,
-      "refunds": {},
-      "review": null,
-      "shipping": null,
-      "source": {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000001",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      },
-      "source_transfer": null,
-      "statement_descriptor": "Zulip Standard",
-      "statement_descriptor_suffix": "Zulip Standard",
-      "status": "succeeded",
-      "transfer_data": null,
-      "transfer_group": null
-    }
-  ],
-  "has_more": false,
-  "object": "list",
-  "url": "/v1/charges"
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Customer.create.1.json

@@ -1,79 +0,0 @@
-{
-  "account_balance": 0,
-  "address": null,
-  "balance": 0,
-  "created": 1000000000,
-  "currency": null,
-  "default_source": "card_NORMALIZED00000000000001",
-  "delinquent": false,
-  "description": "zulip (Zulip Dev)",
-  "discount": null,
-  "email": "hamlet@zulip.com",
-  "id": "cus_NORMALIZED0001",
-  "invoice_prefix": "NORMA01",
-  "invoice_settings": {
-    "custom_fields": null,
-    "default_payment_method": null,
-    "footer": null
-  },
-  "livemode": false,
-  "metadata": {
-    "realm_id": "1",
-    "realm_str": "zulip"
-  },
-  "name": null,
-  "object": "customer",
-  "phone": null,
-  "preferred_locales": [],
-  "shipping": null,
-  "sources": {
-    "data": [
-      {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000001",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      }
-    ],
-    "has_more": false,
-    "object": "list",
-    "total_count": 1,
-    "url": "/v1/customers/cus_NORMALIZED0001/sources"
-  },
-  "subscriptions": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/subscriptions"
-  },
-  "tax_exempt": "none",
-  "tax_ids": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/tax_ids"
-  },
-  "tax_info": null,
-  "tax_info_verification": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Customer.retrieve.1.json

@@ -1,103 +0,0 @@
-{
-  "account_balance": 0,
-  "address": null,
-  "balance": 0,
-  "created": 1000000000,
-  "currency": "usd",
-  "default_source": {
-    "address_city": "Pacific",
-    "address_country": "United States",
-    "address_line1": "Under the sea,",
-    "address_line1_check": "pass",
-    "address_line2": null,
-    "address_state": null,
-    "address_zip": "33333",
-    "address_zip_check": "pass",
-    "brand": "Visa",
-    "country": "US",
-    "customer": "cus_NORMALIZED0001",
-    "cvc_check": "pass",
-    "dynamic_last4": null,
-    "exp_month": 3,
-    "exp_year": 2033,
-    "fingerprint": "NORMALIZED000001",
-    "funding": "credit",
-    "id": "card_NORMALIZED00000000000001",
-    "last4": "4242",
-    "metadata": {},
-    "name": "Ada Starr",
-    "object": "card",
-    "tokenization_method": null
-  },
-  "delinquent": false,
-  "description": "zulip (Zulip Dev)",
-  "discount": null,
-  "email": "hamlet@zulip.com",
-  "id": "cus_NORMALIZED0001",
-  "invoice_prefix": "NORMA01",
-  "invoice_settings": {
-    "custom_fields": null,
-    "default_payment_method": null,
-    "footer": null
-  },
-  "livemode": false,
-  "metadata": {
-    "realm_id": "1",
-    "realm_str": "zulip"
-  },
-  "name": null,
-  "object": "customer",
-  "phone": null,
-  "preferred_locales": [],
-  "shipping": null,
-  "sources": {
-    "data": [
-      {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000001",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      }
-    ],
-    "has_more": false,
-    "object": "list",
-    "total_count": 1,
-    "url": "/v1/customers/cus_NORMALIZED0001/sources"
-  },
-  "subscriptions": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/subscriptions"
-  },
-  "tax_exempt": "none",
-  "tax_ids": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/tax_ids"
-  },
-  "tax_info": null,
-  "tax_info_verification": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Customer.save.1.json

@@ -1,79 +0,0 @@
-{
-  "account_balance": 0,
-  "address": null,
-  "balance": 0,
-  "created": 1000000000,
-  "currency": "usd",
-  "default_source": "card_NORMALIZED00000000000002",
-  "delinquent": false,
-  "description": "zulip (Zulip Dev)",
-  "discount": null,
-  "email": "hamlet@zulip.com",
-  "id": "cus_NORMALIZED0001",
-  "invoice_prefix": "NORMA01",
-  "invoice_settings": {
-    "custom_fields": null,
-    "default_payment_method": null,
-    "footer": null
-  },
-  "livemode": false,
-  "metadata": {
-    "realm_id": "1",
-    "realm_str": "zulip"
-  },
-  "name": null,
-  "object": "customer",
-  "phone": null,
-  "preferred_locales": [],
-  "shipping": null,
-  "sources": {
-    "data": [
-      {
-        "address_city": "Pacific",
-        "address_country": "United States",
-        "address_line1": "Under the sea,",
-        "address_line1_check": "pass",
-        "address_line2": null,
-        "address_state": null,
-        "address_zip": "33333",
-        "address_zip_check": "pass",
-        "brand": "Visa",
-        "country": "US",
-        "customer": "cus_NORMALIZED0001",
-        "cvc_check": "pass",
-        "dynamic_last4": null,
-        "exp_month": 3,
-        "exp_year": 2033,
-        "fingerprint": "NORMALIZED000001",
-        "funding": "credit",
-        "id": "card_NORMALIZED00000000000002",
-        "last4": "4242",
-        "metadata": {},
-        "name": "Ada Starr",
-        "object": "card",
-        "tokenization_method": null
-      }
-    ],
-    "has_more": false,
-    "object": "list",
-    "total_count": 1,
-    "url": "/v1/customers/cus_NORMALIZED0001/sources"
-  },
-  "subscriptions": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/subscriptions"
-  },
-  "tax_exempt": "none",
-  "tax_ids": {
-    "data": [],
-    "has_more": false,
-    "object": "list",
-    "total_count": 0,
-    "url": "/v1/customers/cus_NORMALIZED0001/tax_ids"
-  },
-  "tax_info": null,
-  "tax_info_verification": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Invoice.create.1.json

@@ -1,117 +0,0 @@
-{
-  "account_country": "US",
-  "account_name": "Dev account",
-  "amount_due": 0,
-  "amount_paid": 0,
-  "amount_remaining": 0,
-  "application_fee_amount": null,
-  "attempt_count": 0,
-  "attempted": false,
-  "auto_advance": true,
-  "billing": "charge_automatically",
-  "billing_reason": "manual",
-  "charge": null,
-  "collection_method": "charge_automatically",
-  "created": 1000000000,
-  "currency": "usd",
-  "custom_fields": null,
-  "customer": "cus_NORMALIZED0001",
-  "customer_address": null,
-  "customer_email": "hamlet@zulip.com",
-  "customer_name": null,
-  "customer_phone": null,
-  "customer_shipping": null,
-  "customer_tax_exempt": "none",
-  "customer_tax_ids": [],
-  "default_payment_method": null,
-  "default_source": null,
-  "default_tax_rates": [],
-  "description": "",
-  "discount": null,
-  "due_date": null,
-  "ending_balance": null,
-  "footer": null,
-  "hosted_invoice_url": null,
-  "id": "in_NORMALIZED00000000000001",
-  "invoice_pdf": null,
-  "lines": {
-    "data": [
-      {
-        "amount": 7200,
-        "currency": "usd",
-        "description": "Zulip Standard",
-        "discountable": false,
-        "id": "ii_NORMALIZED00000000000001",
-        "invoice_item": "ii_NORMALIZED00000000000001",
-        "livemode": false,
-        "metadata": {},
-        "object": "line_item",
-        "period": {
-          "end": 1000000000,
-          "start": 1000000000
-        },
-        "plan": null,
-        "proration": false,
-        "quantity": 6,
-        "subscription": null,
-        "tax_amounts": [],
-        "tax_rates": [],
-        "type": "invoiceitem"
-      },
-      {
-        "amount": -7200,
-        "currency": "usd",
-        "description": "Payment (Card ending in 4242)",
-        "discountable": false,
-        "id": "ii_NORMALIZED00000000000002",
-        "invoice_item": "ii_NORMALIZED00000000000002",
-        "livemode": false,
-        "metadata": {},
-        "object": "line_item",
-        "period": {
-          "end": 1000000000,
-          "start": 1000000000
-        },
-        "plan": null,
-        "proration": false,
-        "quantity": 1,
-        "subscription": null,
-        "tax_amounts": [],
-        "tax_rates": [],
-        "type": "invoiceitem"
-      }
-    ],
-    "has_more": false,
-    "object": "list",
-    "total_count": 2,
-    "url": "/v1/invoices/in_NORMALIZED00000000000001/lines"
-  },
-  "livemode": false,
-  "metadata": {},
-  "next_payment_attempt": 1000000000,
-  "number": "NORMALI-0001",
-  "object": "invoice",
-  "paid": false,
-  "payment_intent": null,
-  "period_end": 1000000000,
-  "period_start": 1000000000,
-  "post_payment_credit_notes_amount": 0,
-  "pre_payment_credit_notes_amount": 0,
-  "receipt_number": null,
-  "starting_balance": 0,
-  "statement_descriptor": "Zulip Standard",
-  "status": "draft",
-  "status_transitions": {
-    "finalized_at": null,
-    "marked_uncollectible_at": null,
-    "paid_at": null,
-    "voided_at": null
-  },
-  "subscription": null,
-  "subtotal": 0,
-  "tax": null,
-  "tax_percent": null,
-  "total": 0,
-  "total_tax_amounts": [],
-  "webhooks_delivered_at": null
-}

corporate/tests/stripe_fixtures/attach_discount_to_realm--Invoice.create.2.json

@@ -1,117 +0,0 @@
-{
-  "account_country": "US",
-  "account_name": "Dev account",
-  "amount_due": 0,
-  "amount_paid": 0,
-  "amount_remaining": 0,
-  "application_fee_amount": null,
-  "attempt_count": 0,
-  "attempted": false,
-  "auto_advance": true,
-  "billing": "charge_automatically",
-  "billing_reason": "manual",
-  "charge": null,
-  "collection_method": "charge_automatically",
-  "created": 1000000000,
-  "currency": "usd",
-  "custom_fields": null,
-  "customer": "cus_NORMALIZED0001",
-  "customer_address": null,
-  "customer_email": "hamlet@zulip.com",
-  "customer_name": null,
-  "customer_phone": null,
-  "customer_shipping": null,
-  "customer_tax_exempt": "none",
-  "customer_tax_ids": [],
-  "default_payment_method": null,
-  "default_source": null,
-  "default_tax_rates": [],
-  "description": "",
-  "discount": null,
-  "due_date": null,
-  "ending_balance": null,
-  "footer": null,
-  "hosted_invoice_url": null,
-  "id": "in_NORMALIZED00000000000002",
-  "invoice_pdf": null,
-  "lines": {
-    "data": [
-      {
-        "amount": 36000,
-        "currency": "usd",
-        "description": "Zulip Standard",
-        "discountable": false,
-        "id": "ii_NORMALIZED00000000000003",
-        "invoice_item": "ii_NORMALIZED00000000000003",
-        "livemode": false,
-        "metadata": {},
-        "object": "line_item",
-        "period": {
-          "end": 1000000000,
-          "start": 1000000000
-        },
-        "plan": null,
-        "proration": false,
-        "quantity": 6,
-        "subscription": null,
-        "tax_amounts": [],
-        "tax_rates": [],
-        "type": "invoiceitem"
-      },
-      {
-        "amount": -36000,
-        "currency": "usd",
-        "description": "Payment (Card ending in 4242)",
-        "discountable": false,
-        "id": "ii_NORMALIZED00000000000004",
-        "invoice_item": "ii_NORMALIZED00000000000004",
-        "livemode": false,
-        "metadata": {},
-        "object": "line_item",
-        "period": {
-          "end": 1000000000,
-          "start": 1000000000
-        },
-        "plan": null,
-        "proration": false,
-        "quantity": 1,
-        "subscription": null,
-        "tax_amounts": [],
-        "tax_rates": [],
-        "type": "invoiceitem"
-      }
-    ],
-    "has_more": false,
-    "object": "list",
-    "total_count": 2,
-    "url": "/v1/invoices/in_NORMALIZED00000000000002/lines"
-  },
-  "livemode": false,
-  "metadata": {},
-  "next_payment_attempt": 1000000000,
-  "number": "NORMALI-0002",
-  "object": "invoice",
-  "paid": false,
-  "payment_intent": null,
-  "period_end": 1000000000,
-  "period_start": 1000000000,
-  "post_payment_credit_notes_amount": 0,
-  "pre_payment_credit_notes_amount": 0,
-  "receipt_number": null,
-  "starting_balance": 0,
-  "statement_descriptor": "Zulip Standard",
-  "status": "draft",
-  "status_transitions": {
-    "finalized_at": null,
-    "marked_uncollectible_at": null,
-    "paid_at": null,
-    "voided_at": null
-  },
-  "subscription": null,
-  "subtotal": 0,
-  "tax": null,
-  "tax_percent": null,
-  "total": 0,
-  "total_tax_amounts": [],
-  "webhooks_delivered_at": null
-}