Update changelog for Zulip 1.4.1 and 1.4.2 releases.

Upgrade Django to 1.8.15 with Zulip patches.
puppet: Fix buggy logrotate configuration.
2025-11-04 14:03:30 +00:00 · 2016-09-27 20:09:20 -07:00 · 2016-09-27 20:09:20 -07:00 · 2016-09-27 20:01:17 -07:00 · 2016-09-02 12:30:29 -07:00 · 2016-09-01 21:17:11 -07:00
7702 changed files with 270578 additions and 1097298 deletions
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -1,12 +0,0 @@
 comment: off
 coverage:
  status:
    project:
      default:
        target: auto
        # Codecov has the tendency to report a lot of false negatives,
        # so we basically suppress comments completely.
        threshold: 50%
        base: auto
    patch: off
--- a/.codespellignore
+++ b/.codespellignore
@@ -1,27 +0,0 @@
 te
 ans
 pullrequest
 ist
 cros
 wit
 nwe
 circularly
 ned
 ba
 ressemble
 ser
 sur
 hel
 fpr
 alls
 nd
 ot
 womens
 vise
 falsy
 ro
 derails
 forin
 uper
 slac
 couldn
--- a/.coveralls.yml
+++ b/.coveralls.yml
@@ -0,0 +1,2 @@
 service_name: travis-pro
 repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,24 +0,0 @@
 root = true
 [*]
 end_of_line = lf
 charset = utf-8
 indent_size = 4
 indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 binary_next_line = true  # for shfmt
 switch_case_indent = true  # for shfmt
 [{*.{js,json,ts},check-openapi}]
 max_line_length = 100
 [*.{py,pyi}]
 max_line_length = 110
 [*.{md,svg,rb,pp,yaml,yml}]
 indent_size = 2
 [package.json]
 indent_size = 2
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,15 +0,0 @@
 # This is intended for generated files and vendored third-party files.
 # For our source code, instead of adding files here, consider using
 # specific eslint-disable comments in the files themselves.
 /docs/_build
 /static/generated
 /static/webpack-bundles
 /var/*
 !/var/puppeteer
 /var/puppeteer/*
 !/var/puppeteer/test_credentials.d.ts
 /web/generated
 /web/third
 /zulip-current-venv
 /zulip-py3-venv
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,274 +0,0 @@
 {
    "root": true,
    "env": {
        "es2020": true,
        "node": true
    },
    "extends": [
        "eslint:recommended",
        "plugin:import/errors",
        "plugin:import/warnings",
        "plugin:no-jquery/recommended",
        "plugin:no-jquery/deprecated",
        "plugin:unicorn/recommended",
        "prettier"
    ],
    "parser": "@babel/eslint-parser",
    "parserOptions": {
        "requireConfigFile": false,
        "warnOnUnsupportedTypeScriptVersion": false,
        "sourceType": "unambiguous"
    },
    "plugins": ["formatjs", "no-jquery"],
    "settings": {
        "formatjs": {
            "additionalFunctionNames": ["$t", "$t_html"]
        },
        "no-jquery": {
            "collectionReturningPlugins": {
                "expectOne": "always"
            },
            "variablePattern": "^\\$(?!t$|t_html$)."
        }
    },
    "reportUnusedDisableDirectives": true,
    "rules": {
        "array-callback-return": "error",
        "arrow-body-style": "error",
        "block-scoped-var": "error",
        "consistent-return": "error",
        "curly": "error",
        "dot-notation": "error",
        "eqeqeq": "error",
        "formatjs/enforce-default-message": ["error", "literal"],
        "formatjs/enforce-placeholders": [
            "error",
            {"ignoreList": ["b", "code", "em", "i", "kbd", "p", "strong"]}
        ],
        "formatjs/no-id": "error",
        "guard-for-in": "error",
        "import/extensions": "error",
        "import/first": "error",
        "import/newline-after-import": "error",
        "import/no-self-import": "error",
        "import/no-unresolved": "off",
        "import/no-useless-path-segments": "error",
        "import/order": ["error", {"alphabetize": {"order": "asc"}, "newlines-between": "always"}],
        "import/unambiguous": "error",
        "lines-around-directive": "error",
        "new-cap": "error",
        "no-alert": "error",
        "no-array-constructor": "error",
        "no-bitwise": "error",
        "no-caller": "error",
        "no-catch-shadow": "error",
        "no-constant-condition": ["error", {"checkLoops": false}],
        "no-div-regex": "error",
        "no-duplicate-imports": "error",
        "no-else-return": "error",
        "no-eq-null": "error",
        "no-eval": "error",
        "no-implicit-coercion": "error",
        "no-implied-eval": "error",
        "no-inner-declarations": "off",
        "no-iterator": "error",
        "no-jquery/no-constructor-attributes": "error",
        "no-jquery/no-parse-html-literal": "error",
        "no-label-var": "error",
        "no-labels": "error",
        "no-loop-func": "error",
        "no-multi-str": "error",
        "no-native-reassign": "error",
        "no-new-func": "error",
        "no-new-object": "error",
        "no-new-wrappers": "error",
        "no-octal-escape": "error",
        "no-plusplus": "error",
        "no-proto": "error",
        "no-return-assign": "error",
        "no-script-url": "error",
        "no-self-compare": "error",
        "no-sync": "error",
        "no-throw-literal": "error",
        "no-undef-init": "error",
        "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
        "no-unused-expressions": "error",
        "no-unused-vars": ["error", {"ignoreRestSiblings": true}],
        "no-use-before-define": ["error", {"functions": false}],
        "no-useless-concat": "error",
        "no-useless-constructor": "error",
        "no-var": "error",
        "object-shorthand": ["error", "always", {"avoidExplicitReturnArrows": true}],
        "one-var": ["error", "never"],
        "prefer-arrow-callback": "error",
        "prefer-const": ["error", {"ignoreReadBeforeAssign": true}],
        "radix": "error",
        "sort-imports": ["error", {"ignoreDeclarationSort": true}],
        "spaced-comment": ["error", "always", {"markers": ["/"]}],
        "strict": "error",
        "unicorn/consistent-function-scoping": "off",
        "unicorn/explicit-length-check": "off",
        "unicorn/filename-case": "off",
        "unicorn/no-await-expression-member": "off",
        "unicorn/no-negated-condition": "off",
        "unicorn/no-null": "off",
        "unicorn/no-process-exit": "off",
        "unicorn/no-useless-undefined": "off",
        "unicorn/numeric-separators-style": "off",
        "unicorn/prefer-module": "off",
        "unicorn/prefer-node-protocol": "off",
        "unicorn/prefer-ternary": "off",
        "unicorn/prefer-top-level-await": "off",
        "unicorn/prevent-abbreviations": "off",
        "unicorn/switch-case-braces": "off",
        "valid-typeof": ["error", {"requireStringLiterals": true}],
        "yoda": "error"
    },
    "overrides": [
        {
            "files": ["web/tests/**"],
            "rules": {
                "no-jquery/no-selector-prop": "off"
            }
        },
        {
            "files": ["web/e2e-tests/**"],
            "globals": {
                "zulip_test": false
            }
        },
        {
            "files": ["web/src/**"],
            "globals": {
                "StripeCheckout": false
            }
        },
        {
            "files": ["**/*.ts"],
            "extends": [
                "plugin:@typescript-eslint/recommended",
                "plugin:@typescript-eslint/recommended-requiring-type-checking",
                "plugin:@typescript-eslint/strict",
                "plugin:import/typescript"
            ],
            "parserOptions": {
                "project": "tsconfig.json"
            },
            "settings": {
                "import/resolver": {
                    "node": {
                        "extensions": [".ts", ".d.ts", ".js"] // https://github.com/import-js/eslint-plugin-import/issues/2267
                    }
                }
            },
            "globals": {
                "JQuery": false
            },
            "rules": {
                // Disable base rule to avoid conflict
                "no-duplicate-imports": "off",
                "no-use-before-define": "off",
                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
                "@typescript-eslint/consistent-type-imports": "error",
                "@typescript-eslint/explicit-function-return-type": [
                    "error",
                    {"allowExpressions": true}
                ],
                "@typescript-eslint/member-ordering": "error",
                "@typescript-eslint/no-duplicate-imports": "error",
                "@typescript-eslint/no-non-null-assertion": "off",
                "@typescript-eslint/no-parameter-properties": "error",
                "@typescript-eslint/no-unnecessary-condition": "off",
                "@typescript-eslint/no-unnecessary-qualifier": "error",
                "@typescript-eslint/no-unsafe-argument": "off",
                "@typescript-eslint/no-unsafe-assignment": "off",
                "@typescript-eslint/no-unsafe-call": "off",
                "@typescript-eslint/no-unsafe-member-access": "off",
                "@typescript-eslint/no-unsafe-return": "off",
                "@typescript-eslint/no-unused-vars": ["error", {"ignoreRestSiblings": true}],
                "@typescript-eslint/no-use-before-define": ["error", {"functions": false}],
                "@typescript-eslint/promise-function-async": "error",
                "import/no-cycle": "error",
                "no-undef": "error"
            }
        },
        {
            "files": ["**/*.d.ts"],
            "rules": {
                "import/unambiguous": "off"
            }
        },
        {
            "files": ["web/e2e-tests/**", "web/tests/**"],
            "globals": {
                "CSS": false,
                "document": false,
                "navigator": false,
                "window": false
            },
            "rules": {
                "formatjs/no-id": "off",
                "new-cap": "off",
                "no-sync": "off",
                "unicorn/prefer-prototype-methods": "off"
            }
        },
        {
            "files": ["web/debug-require.js"],
            "env": {
                "browser": true,
                "es2020": false
            },
            "rules": {
                // Don’t require ES features that PhantomJS doesn’t support
                // TODO: Toggle these settings now that we don't use PhantomJS
                "no-var": "off",
                "object-shorthand": "off",
                "prefer-arrow-callback": "off"
            }
        },
        {
            "files": ["web/shared/**", "web/src/**", "web/third/**"],
            "env": {
                "browser": true,
                "node": false
            },
            "globals": {
                "ZULIP_VERSION": false
            },
            "rules": {
                "no-console": "error"
            },
            "settings": {
                "import/resolver": {
                    "webpack": {
                        "config": "./web/webpack.config.ts"
                    }
                }
            }
        },
        {
            "files": ["web/shared/**"],
            "env": {
                "browser": false,
                "shared-node-browser": true
            },
            "rules": {
                "import/no-restricted-paths": [
                    "error",
                    {
                        "zones": [
                            {
                                "target": "./web/shared",
                                "from": ".",
                                "except": ["./node_modules", "./web/shared"]
                            }
                        ]
                    }
                ],
                "unicorn/prefer-string-replace-all": "off"
            }
        }
    ]
 }
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,32 +1,21 @@
-# DIFFS: Noise suppression.
+.gitignore export-ignore
-#
+.gitattributes export-ignore
-# Suppress noisy generated files in diffs.
+/analytics export-ignore
-# (When you actually want to see these diffs, use `git diff -a`.)
+/assets export-ignore
-
+/bots export-ignore
-# Large test fixtures:
+/corporate export-ignore
-corporate/tests/stripe_fixtures/*.json -diff
+/static export-ignore
-
+/tools export-ignore
-
+/zilencer export-ignore
-# FORMATTING
+/templates/analytics export-ignore
-
+/templates/corporate export-ignore
-# Maintain LF (Unix-style) newlines in text files.
+/templates/zilencer export-ignore
-*   text=auto eol=lf
+/puppet/zulip_internal export-ignore
-
+/zproject/local_settings.py export-ignore
-# Make sure various media files never get somehow auto-detected as text
+/zproject/test_settings.py export-ignore
-# and then newline-converted.
+/zerver/fixtures export-ignore
-*.gif binary
+/zerver/tests export-ignore
-*.jpg binary
+/frontend_tests export-ignore
-*.jpeg binary
+/node_modules export-ignore
-*.eot binary
+/humbug export-ignore
-*.woff binary
+/locale export-ignore
 *.woff2 binary
 *.svg binary
 *.ttf binary
 *.png binary
 *.otf binary
 *.tif binary
 *.ogg binary
 *.bson binary
 *.bmp binary
 *.mp3 binary
 *.pdf binary
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,3 +0,0 @@
 github: zulip
 patreon: zulip
 open_collective: zulip
--- a/.github/ISSUE_TEMPLATE/1_discussed_on_czo.md
+++ b/.github/ISSUE_TEMPLATE/1_discussed_on_czo.md
@@ -1,10 +0,0 @@
 ---
 name: Issue discussed in the Zulip development community
 about: Bug report, feature or improvement already discussed on chat.zulip.org.
 ---
 <!-- Issue description -->
 <!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
 CZO thread
--- a/.github/ISSUE_TEMPLATE/2_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/2_bug_report.md
@@ -1,17 +0,0 @@
 ---
 name: Bug report
 about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
 labels: ["bug"]
 ---
 <!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
 <!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
 **Zulip Server and web app version:**
 - [ ] Zulip Cloud (`*.zulipchat.com`)
 - [ ] Zulip Server 7.0+
 - [ ] Zulip Server 6.0+
 - [ ] Zulip Server 5.0 or older
 - [ ] Other or not sure
--- a/.github/ISSUE_TEMPLATE/3_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_feature_request.md
@@ -1,6 +0,0 @@
 ---
 name: Feature or improvement request
 about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
 ---
 <!-- Describe the proposal, including how it would help you or your organization. -->
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,14 +0,0 @@
 blank_issues_enabled: true
 contact_links:
  - name: Possible bug
    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
    about: Report unexpected behavior that may be a bug.
  - name: Feature suggestion or feedback
    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
    about: Start a discussion about your idea for improving Zulip.
  - name: Issue with running or upgrading a Zulip server
    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
  - name: Other support requests and sales questions
    url: https://zulip.com/help/contact-support
    about: Contact us — we're happy to help!
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,43 +0,0 @@
 <!-- Describe your pull request here.-->
 Fixes: <!-- Issue link, or clear description.-->
 <!-- If the PR makes UI changes, always include one or more still screenshots to demonstrate your changes. If it seems helpful, add a screen capture of the new functionality as well.
 Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
 -->
 **Screenshots and screen captures:**
 <details>
 <summary>Self-review checklist</summary>
 <!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
 https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
 <!-- Once you create the PR, check off all the steps below that you have completed.
 If any of these steps are not relevant or you have not completed, leave them unchecked.-->
 - [ ] [Self-reviewed](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code) the changes for clarity and maintainability
      (variable names, code reuse, readability, etc.).
 Communicate decisions, questions, and potential concerns.
 - [ ] Explains differences from previous plans (e.g., issue description).
 - [ ] Highlights technical choices and bugs encountered.
 - [ ] Calls out remaining decisions and concerns.
 - [ ] Automated tests verify logic where appropriate.
 Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
 - [ ] Each commit is a coherent idea.
 - [ ] Commit message(s) explain reasoning and motivation for changes.
 Completed manual review and testing of the following:
 - [ ] Visual appearance of the changes.
 - [ ] Responsiveness and internationalization.
 - [ ] Strings and tooltips.
 - [ ] End-to-end functionality of buttons, interactions and flows.
 - [ ] Corner cases, error conditions, and easily imagined bugs.
 </details>
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,40 +0,0 @@
 name: "Code scanning"
 on:
  push:
    branches: ["*.x", chat.zulip.org, main]
    tags: ["*"]
  pull_request:
    branches: ["*.x", chat.zulip.org, main]
  workflow_dispatch:
 concurrency:
  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
  cancel-in-progress: true
 permissions:
  contents: read
 jobs:
  CodeQL:
    permissions:
      actions: read # for github/codeql-action/init to get workflow details
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/analyze to upload SARIF results
    if: ${{!github.event.repository.private}}
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v3
      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        # Override language selection by uncommenting this and choosing your languages
        # with:
        #   languages: go, javascript, csharp, python, cpp, java
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2
--- a/.github/workflows/production-suite.yml
+++ b/.github/workflows/production-suite.yml
@@ -1,326 +0,0 @@
 name: Zulip production suite
 on:
  push:
    branches: ["*.x", chat.zulip.org, main]
    tags: ["*"]
  pull_request:
    paths:
      - .github/workflows/production-suite.yml
      - "**/migrations/**"
      - manage.py
      - pnpm-lock.yaml
      - puppet/**
      - requirements/**
      - scripts/**
      - tools/**
      - web/babel.config.js
      - web/postcss.config.js
      - web/third/**
      - web/webpack.config.ts
      - zerver/worker/queue_processors.py
      - zerver/lib/push_notifications.py
      - zerver/decorator.py
      - zproject/**
  workflow_dispatch:
 concurrency:
  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
  cancel-in-progress: true
 defaults:
  run:
    shell: bash
 permissions:
  contents: read
 jobs:
  production_build:
    # This job builds a release tarball from the current commit, which
    # will be used for all of the following install/upgrade tests.
    name: Ubuntu 20.04 production build
    runs-on: ubuntu-latest
    # Docker images are built from 'tools/ci/Dockerfile'; the comments at
    # the top explain how to build and upload these images.
    # Ubuntu 20.04 ships with Python 3.8.10.
    container: zulip/ci:focal
    steps:
      - name: Add required permissions
        run: |
          # The checkout actions doesn't clone to ~/zulip or allow
          # us to use the path option to clone outside the current
          # /__w/zulip/zulip directory. Since this directory is owned
          # by root we need to change it's ownership to allow the
          # github user to clone the code here.
          # Note: /__w/ is a docker volume mounted to $GITHUB_WORKSPACE
          # which is /home/runner/work/.
          sudo chown -R github .
          # This is the GitHub Actions specific cache directory the
          # the current github user must be able to access for the
          # cache action to work. It is owned by root currently.
          sudo chmod -R 0777 /__w/_temp/
      - uses: actions/checkout@v3
      - name: Create cache directories
        run: |
          dirs=(/srv/zulip-{venv,emoji}-cache)
          sudo mkdir -p "${dirs[@]}"
          sudo chown -R github "${dirs[@]}"
      - name: Restore pnpm store
        uses: actions/cache@v3
        with:
          path: ~/.local/share/pnpm/store
          key: v1-pnpm-store-focal-${{ hashFiles('pnpm-lock.yaml') }}
      - name: Restore python cache
        uses: actions/cache@v3
        with:
          path: /srv/zulip-venv-cache
          key: v1-venv-focal-${{ hashFiles('requirements/dev.txt') }}
          restore-keys: v1-venv-focal
      - name: Restore emoji cache
        uses: actions/cache@v3
        with:
          path: /srv/zulip-emoji-cache
          key: v1-emoji-focal-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
          restore-keys: v1-emoji-focal
      - name: Build production tarball
        run: ./tools/ci/production-build
      - name: Upload production build artifacts for install jobs
        uses: actions/upload-artifact@v3
        with:
          name: production-tarball
          path: /tmp/production-build
          retention-days: 1
      - name: Generate failure report string
        id: failure_report_string
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
      - name: Report status to CZO
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        uses: zulip/github-actions-zulip/send-message@v1
        with:
          api-key: ${{ secrets.ZULIP_BOT_KEY }}
          email: "github-actions-bot@chat.zulip.org"
          organization-url: "https://chat.zulip.org"
          to: "automated testing"
          topic: ${{ steps.failure_report_string.outputs.topic }}
          type: "stream"
          content: ${{ steps.failure_report_string.outputs.content }}
  production_install:
    # This job installs the server release tarball built above on a
    # range of platforms, and does some basic health checks on the
    # resulting installer Zulip server.
    strategy:
      fail-fast: false
      matrix:
        include:
          # Docker images are built from 'tools/ci/Dockerfile'; the comments at
          # the top explain how to build and upload these images.
          - docker_image: zulip/ci:focal
            name: Ubuntu 20.04 production install and PostgreSQL upgrade with pgroonga
            os: focal
            extra-args: ""
          - docker_image: zulip/ci:jammy
            name: Ubuntu 22.04 production install
            os: jammy
            extra-args: ""
          - docker_image: zulip/ci:bullseye
            name: Debian 11 production install with custom db name and user
            os: bullseye
            extra-args: --test-custom-db
          - docker_image: zulip/ci:bookworm
            name: Debian 12 production install
            os: bookworm
            extra-args: ""
    name: ${{ matrix.name  }}
    container:
      image: ${{ matrix.docker_image }}
      options: --init
    runs-on: ubuntu-latest
    needs: production_build
    steps:
      - name: Download built production tarball
        uses: actions/download-artifact@v3
        with:
          name: production-tarball
          path: /tmp
      - name: Add required permissions and setup
        run: |
          # This is the GitHub Actions specific cache directory the
          # the current github user must be able to access for the
          # cache action to work. It is owned by root currently.
          sudo chmod -R 0777 /__w/_temp/
          # Since actions/download-artifact@v2 loses all the permissions
          # of the tarball uploaded by the upload artifact fix those.
          chmod +x /tmp/production-upgrade-pg
          chmod +x /tmp/production-pgroonga
          chmod +x /tmp/production-install
          chmod +x /tmp/production-verify
          chmod +x /tmp/generate-failure-message
      - name: Create cache directories
        run: |
          dirs=(/srv/zulip-{venv,emoji}-cache)
          sudo mkdir -p "${dirs[@]}"
          sudo chown -R github "${dirs[@]}"
      - name: Restore pnpm store
        uses: actions/cache@v3
        with:
          path: ~/.local/share/pnpm/store
          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('/tmp/pnpm-lock.yaml') }}
      - name: Install production
        run: sudo /tmp/production-install ${{ matrix.extra-args }}
      - name: Verify install
        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
      - name: Install pgroonga
        if: ${{ matrix.os == 'focal' }}
        run: sudo /tmp/production-pgroonga
      - name: Verify install after installing pgroonga
        if: ${{ matrix.os == 'focal' }}
        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
      - name: Upgrade postgresql
        if: ${{ matrix.os == 'focal' }}
        run: sudo /tmp/production-upgrade-pg
      - name: Verify install after upgrading postgresql
        if: ${{ matrix.os == 'focal' }}
        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
      - name: Generate failure report string
        id: failure_report_string
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
      - name: Report status to CZO
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        uses: zulip/github-actions-zulip/send-message@v1
        with:
          api-key: ${{ secrets.ZULIP_BOT_KEY }}
          email: "github-actions-bot@chat.zulip.org"
          organization-url: "https://chat.zulip.org"
          to: "automated testing"
          topic: ${{ steps.failure_report_string.outputs.topic }}
          type: "stream"
          content: ${{ steps.failure_report_string.outputs.content }}
  production_upgrade:
    # The production upgrade job starts with a container with a
    # previous Zulip release installed, and attempts to upgrade it to
    # the release tarball built for the current commit being tested.
    #
    # This is intended to catch bugs that result in the upgrade
    # process failing.
    strategy:
      fail-fast: false
      matrix:
        include:
          # Docker images are built from 'tools/ci/Dockerfile.prod'; the comments at
          # the top explain how to build and upload these images.
          - docker_image: zulip/ci:focal-3.2
            name: 3.2 Version Upgrade
            os: focal
          - docker_image: zulip/ci:bullseye-4.2
            name: 4.2 Version Upgrade
            os: bullseye
          - docker_image: zulip/ci:bullseye-5.0
            name: 5.0 Version Upgrade
            os: bullseye
          - docker_image: zulip/ci:bullseye-6.0
            name: 6.0 Version Upgrade
            os: bullseye
    name: ${{ matrix.name  }}
    container:
      image: ${{ matrix.docker_image }}
      options: --init
    runs-on: ubuntu-latest
    needs: production_build
    steps:
      - name: Download built production tarball
        uses: actions/download-artifact@v3
        with:
          name: production-tarball
          path: /tmp
      - name: Add required permissions and setup
        run: |
          # This is the GitHub Actions specific cache directory the
          # the current github user must be able to access for the
          # cache action to work. It is owned by root currently.
          sudo chmod -R 0777 /__w/_temp/
          # Since actions/download-artifact@v2 loses all the permissions
          # of the tarball uploaded by the upload artifact fix those.
          chmod +x /tmp/production-upgrade
          chmod +x /tmp/production-verify
          chmod +x /tmp/generate-failure-message
      - name: Create cache directories
        run: |
          dirs=(/srv/zulip-{venv,emoji}-cache)
          sudo mkdir -p "${dirs[@]}"
          sudo chown -R github "${dirs[@]}"
      - name: Temporarily bootstrap PostgreSQL upgrades
        # https://chat.zulip.org/#narrow/stream/43-automated-testing/topic/postgres.20client.20upgrade.20failures/near/1640444
        # On Debian, there is an ordering issue with post-install maintainer
        # scripts when postgresql-client-common is upgraded at the same time as
        # postgresql-client and postgresql-client-15.  Upgrade just
        # postgresql-client-common first, so the main upgrade process can
        # succeed.  This is a _temporary_ work-around to improve CI signal, as
        # the failure does represent a real failure that production systems may
        # encounter.
        run: sudo apt-get update && sudo apt-get install -y --only-upgrade postgresql-client-common
      - name: Upgrade production
        run: sudo /tmp/production-upgrade
        # TODO: We should be running production-verify here, but it
        # doesn't pass yet.
        #
        # - name: Verify install
        #   run: sudo /tmp/production-verify
      - name: Generate failure report string
        id: failure_report_string
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
      - name: Report status to CZO
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        uses: zulip/github-actions-zulip/send-message@v1
        with:
          api-key: ${{ secrets.ZULIP_BOT_KEY }}
          email: "github-actions-bot@chat.zulip.org"
          organization-url: "https://chat.zulip.org"
          to: "automated testing"
          topic: ${{ steps.failure_report_string.outputs.topic }}
          type: "stream"
          content: ${{ steps.failure_report_string.outputs.content }}
--- a/.github/workflows/update-oneclick-apps.yml
+++ b/.github/workflows/update-oneclick-apps.yml
@@ -1,27 +0,0 @@
 name: Update one click apps
 on:
  release:
    types: [published]
 permissions:
  contents: read
 jobs:
  update-digitalocean-oneclick-app:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Update DigitalOcean one click app
        env:
          DIGITALOCEAN_API_KEY: ${{ secrets.ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY }}
          ZULIP_API_KEY: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_API_KEY }}
          ZULIP_EMAIL: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_EMAIL }}
          ZULIP_SITE: https://chat.zulip.org
          ONE_CLICK_ACTION_STREAM: kandra ops
          PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC: 30
          RELEASE_VERSION: ${{ github.event.release.tag_name }}
        run: |
          export PATH="$HOME/.local/bin:$PATH"
          git clone https://github.com/zulip/marketplace-partners
          pip3 install python-digitalocean zulip fab-classic PyNaCl
          echo $PATH
          python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py
--- a/.github/workflows/zulip-ci.yml
+++ b/.github/workflows/zulip-ci.yml
@@ -1,260 +0,0 @@
 # NOTE: Everything test in this file should be in `tools/test-all`.  If there's a
 # reason not to run it there, it should be there as a comment
 # explaining why.
 name: Zulip CI
 on:
  push:
    branches: ["*.x", chat.zulip.org, main]
    tags: ["*"]
  pull_request:
  workflow_dispatch:
 concurrency:
  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
  cancel-in-progress: true
 defaults:
  run:
    shell: bash
 permissions:
  contents: read
 jobs:
  tests:
    strategy:
      fail-fast: false
      matrix:
        include:
          # Base images are built using `tools/ci/Dockerfile.prod.template`.
          # The comments at the top explain how to build and upload these images.
          # Ubuntu 20.04 ships with Python 3.8.10.
          - docker_image: zulip/ci:focal
            name: Ubuntu 20.04 (Python 3.8, backend + frontend)
            os: focal
            include_documentation_tests: false
            include_frontend_tests: true
          # Debian 11 ships with Python 3.9.2.
          - docker_image: zulip/ci:bullseye
            name: Debian 11 (Python 3.9, backend + documentation)
            os: bullseye
            include_documentation_tests: true
            include_frontend_tests: false
          # Ubuntu 22.04 ships with Python 3.10.4.
          - docker_image: zulip/ci:jammy
            name: Ubuntu 22.04 (Python 3.10, backend)
            os: jammy
            include_documentation_tests: false
            include_frontend_tests: false
          # Debian 12 ships with Python 3.11.2.
          - docker_image: zulip/ci:bookworm
            name: Debian 12 (Python 3.11, backend)
            os: bookworm
            include_documentation_tests: false
            include_frontend_tests: false
    runs-on: ubuntu-latest
    name: ${{ matrix.name }}
    container: ${{ matrix.docker_image }}
    env:
      # GitHub Actions sets HOME to /github/home which causes
      # problem later in provision and frontend test that runs
      # tools/setup/postgresql-init-dev-db because of the .pgpass
      # location. PostgreSQL (psql) expects .pgpass to be at
      # /home/github/.pgpass and setting home to `/home/github/`
      # ensures it written there because we write it to ~/.pgpass.
      HOME: /home/github/
    steps:
      - uses: actions/checkout@v3
      - name: Create cache directories
        run: |
          dirs=(/srv/zulip-{venv,emoji}-cache)
          sudo mkdir -p "${dirs[@]}"
          sudo chown -R github "${dirs[@]}"
      - name: Restore pnpm store
        uses: actions/cache@v3
        with:
          path: ~/.local/share/pnpm/store
          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
      - name: Restore python cache
        uses: actions/cache@v3
        with:
          path: /srv/zulip-venv-cache
          key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/dev.txt') }}
          restore-keys: v1-venv-${{ matrix.os }}
      - name: Restore emoji cache
        uses: actions/cache@v3
        with:
          path: /srv/zulip-emoji-cache
          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json', 'tools/setup/emoji/build_emoji', 'tools/setup/emoji/emoji_setup_utils.py', 'tools/setup/emoji/emoji_names.py', 'package.json') }}
          restore-keys: v1-emoji-${{ matrix.os }}
      - name: Install dependencies
        run: |
          # This is the main setup job for the test suite
          ./tools/ci/setup-backend --skip-dev-db-build
          scripts/lib/clean_unused_caches.py --verbose --threshold=0
      - name: Run tools test
        run: |
          source tools/ci/activate-venv
          ./tools/test-tools
      - name: Run Codespell lint
        run: |
          source tools/ci/activate-venv
          ./tools/run-codespell
      - name: Run backend lint
        run: |
          source tools/ci/activate-venv
          echo "Test suite is running under $(python --version)."
          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
      - name: Run frontend lint
        if: ${{ matrix.include_frontend_tests }}
        run: |
          source tools/ci/activate-venv
          ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
      - name: Run backend tests
        run: |
          source tools/ci/activate-venv
          ./tools/test-backend --coverage --xml-report --no-html-report --include-webhooks --no-cov-cleanup --ban-console-output
      - name: Run mypy
        run: |
          source tools/ci/activate-venv
          # We run mypy after the backend tests so we get output from the
          # backend tests, which tend to uncover more serious problems, first.
          ./tools/run-mypy --version
          ./tools/run-mypy
      - name: Run miscellaneous tests
        run: |
          source tools/ci/activate-venv
          # Currently our compiled requirements files will differ for different
          # Python versions, so we will run test-locked-requirements only on the
          # platform with the oldest one.
          # ./tools/test-locked-requirements
          # ./tools/test-run-dev  # https://github.com/zulip/zulip/pull/14233
          #
          # This test has been persistently flaky at like 1% frequency, is slow,
          # and is for a very specific single feature, so we don't run it by default:
          # ./tools/test-queue-worker-reload
          ./tools/test-migrations
          ./tools/setup/optimize-svg --check
          ./tools/setup/generate_integration_bots_avatars.py --check-missing
          ./tools/ci/check-executables
          # Ban check-database-compatibility from transitively
          # relying on static/generated, because it might not be
          # up-to-date at that point in upgrade-zulip-stage-2.
          chmod 000 static/generated web/generated
          ./scripts/lib/check-database-compatibility
          chmod 755 static/generated web/generated
      - name: Run documentation and api tests
        if: ${{ matrix.include_documentation_tests }}
        run: |
          source tools/ci/activate-venv
          # In CI, we only test links we control in test-documentation to avoid flakes
          ./tools/test-documentation --skip-external-links
          ./tools/test-help-documentation --skip-external-links
          ./tools/test-api
      - name: Run node tests
        if: ${{ matrix.include_frontend_tests }}
        run: |
          source tools/ci/activate-venv
          # Run the node tests first, since they're fast and deterministic
          ./tools/test-js-with-node --coverage --parallel=1
      - name: Check schemas
        if: ${{ matrix.include_frontend_tests }}
        run: |
          source tools/ci/activate-venv
          # Check that various schemas are consistent. (is fast)
          ./tools/check-schemas
      - name: Check capitalization of strings
        if: ${{ matrix.include_frontend_tests }}
        run: |
          source tools/ci/activate-venv
          ./manage.py makemessages --locale en
          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
      - name: Run puppeteer tests
        if: ${{ matrix.include_frontend_tests }}
        run: |
          source tools/ci/activate-venv
          ./tools/test-js-with-puppeteer
      - name: Check pnpm dedupe
        if: ${{ matrix.include_frontend_tests }}
        run: pnpm dedupe --check
      - name: Check for untracked files
        run: |
          source tools/ci/activate-venv
          # This final check looks for untracked files that may have been
          # created by test-backend or provision.
          untracked="$(git ls-files --exclude-standard --others)"
          if [ -n "$untracked" ]; then
              printf >&2 "Error: untracked files:\n%s\n" "$untracked"
              exit 1
          fi
      - name: Test locked requirements
        if: ${{ matrix.os == 'focal' }}
        run: |
          . /srv/zulip-py3-venv/bin/activate && \
          ./tools/test-locked-requirements
      - name: Upload coverage reports
        # Only upload coverage when both frontend and backend
        # tests are run.
        if: ${{ matrix.include_frontend_tests }}
        uses: codecov/codecov-action@v3
        with:
          files: var/coverage.xml,var/node-coverage/lcov.info
      - name: Store Puppeteer artifacts
        # Upload these on failure, as well
        if: ${{ always() && matrix.include_frontend_tests }}
        uses: actions/upload-artifact@v3
        with:
          name: puppeteer
          path: ./var/puppeteer
          retention-days: 60
      - name: Check development database build
        run: ./tools/ci/setup-backend
      - name: Generate failure report string
        id: failure_report_string
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
      - name: Report status to CZO
        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
        uses: zulip/github-actions-zulip/send-message@v1
        with:
          api-key: ${{ secrets.ZULIP_BOT_KEY }}
          email: "github-actions-bot@chat.zulip.org"
          organization-url: "https://chat.zulip.org"
          to: "automated testing"
          topic: ${{ steps.failure_report_string.outputs.topic }}
          type: "stream"
          content: ${{ steps.failure_report_string.outputs.content }}
--- a/.gitignore
+++ b/.gitignore
@@ -1,88 +1,26 @@
 # Quick format and style primer:
 #
 #  * If a pattern is meant only for a specific location, it should have a
 #    leading slash, like `/staticfiles.json`.
 #    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
 #      will do, but this makes a confusing pattern.  Adding a leading slash
 #      is clearer.
 #
 #  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
 #    match in any subdirectory.
 #
 #  * Subdirectories with several internal things to ignore get their own
 #    `.gitignore` files.
 #
 #  * Comments must be on their own line.  (Otherwise they don't work.)
 #
 # See `git help ignore` for details on the format.
 ## Config files for the dev environment
 /zproject/dev-secrets.conf
 /tools/conf.ini
 /tools/custom_provision
 /tools/droplets/conf.ini
 ## Byproducts of setting up and using the dev environment
 *.pyc
 package-lock.json
 /.vagrant
 /var/*
 !/var/puppeteer
 /var/puppeteer/*
 !/var/puppeteer/test_credentials.d.ts
 /.dmypy.json
 /.ruff_cache
 # Generated i18n data
 /locale/en
 /locale/language_options.json
 /locale/language_name_map.json
 /locale/*/mobile.json
 # Static build
 *.mo
 npm-debug.log
 /.pnpm-store
 /node_modules
 /prod-static
 /staticfiles.json
 /webpack-stats-production.json
 zulip-git-version
 # Test / analysis tools
 .coverage
 ## Files (or really symlinks) created in a prod deployment
 /zproject/prod_settings.py
 /zulip-current-venv
 /zulip-py3-venv
 ## Files left by various editors and local environments
 # (Ideally these should be in everyone's respective personal gitignore files.)
 *~
 /prod-static
 /errors/*
 *.sw[po]
-.idea
+*.DS_Store
 stats/
 .kdev4
 .idea
 zulip.kdev4
 coverage/
 .coverage
 /queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
-*.sublime-project
+.vagrant
-*.sublime-workspace
+/zproject/dev-secrets.conf
-*.DS_Store
+static/js/bundle.js
-# VS Code. Avoid checking in .vscode in general, while still specifying
+static/third/gemoji/
-# recommended extensions for working with this repository.
+static/third/zxcvbn/
-/.vscode/**/*
+static/locale/language_options.json
-!/.vscode/extensions.json
+node_modules
-# .cache/ is generated by VS Code test runner
+npm-debug.log
-.cache/
+*.mo
-.eslintcache
+var/*
 # Core dump files
 core
 ## Miscellaneous
 # (Ideally this section is empty.)
 .transifexrc
--- a/.gitlint
+++ b/.gitlint
@@ -1,13 +0,0 @@
 [general]
 ignore=title-trailing-punctuation, body-min-length, body-is-missing
 extra-path=tools/lib/gitlint_rules.py
 [title-match-regex]
 regex=^(.+:\ )?[A-Z].+\.$
 [title-max-length]
 line-length=72
 [body-max-line-length]
 line-length=76
--- a/.mailmap
+++ b/.mailmap
@@ -1,131 +0,0 @@
 # This file teaches `git log` and friends the canonical names
 # and email addresses to use for our contributors.
 #
 # For details on the format, see:
 #   https://git.github.io/htmldocs/gitmailmap.html
 #
 # Handy commands for examining or adding to this file:
 #
 #     # shows all names/emails after mapping, sorted:
 #   $ git shortlog -es | sort -k2
 #
 #     # shows raw names/emails, filtered by mapped name:
 #   $ git log --format='%an %ae' --author=$NAME | uniq -c
 acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
 acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
 acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
 Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
 Adam Benesh <Adam.Benesh@gmail.com>
 Adarsh Tiwari <xoldyckk@gmail.com>
 Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
 Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
 Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
 Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
 Aman Agrawal <amanagr@zulip.com>
 Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
 Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
 Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
 Aryan Shridhar <aryanshridhar7@gmail.com>
 aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
 Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
 Austin Riba <austin@zulip.com> <austin@m51.io>
 BIKI DAS <bikid475@gmail.com>
 Brijmohan Siyag <brijsiyag@gmail.com>
 Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
 Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
 Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
 Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
 Danny Su <contact@dannysu.com> <opensource@emailengine.org>
 Dinesh <chdinesh1089@gmail.com>
 Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
 Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
 Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
 Evy Kassirer <evy.kassirer@gmail.com>
 Evy Kassirer <evy.kassirer@gmail.com> <evykassirer@users.noreply.github.com>
 Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
 Greg Price <greg@zulip.com> <gnprice@gmail.com>
 Greg Price <greg@zulip.com> <greg@zulipchat.com>
 Greg Price <greg@zulip.com> <price@mit.edu>
 Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
 Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
 Jai soni <jai_s@me.iitr.ac.in>
 Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
 Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
 Karl Stolley <karl@zulip.com> <karl@stolley.dev>
 Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
 Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
 Kevin Scott <kevin.scott.98@gmail.com>
 Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
 Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
 Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
 Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
 Matt Keller <matt@zulip.com>
 Matt Keller <matt@zulip.com> <m@cognusion.com>
 m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
 m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
 Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
 nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
 Palash Baderia <palash.baderia@outlook.com>
 Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
 Palash Raghuwanshi <singhpalash0@gmail.com>
 Parth <mittalparth22@gmail.com>
 Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
 Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
 Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
 Rein Zustand (rht) <rhtbot@protonmail.com>
 Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
 Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
 Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
 Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Rixant Rokaha <rixantrokaha@gmail.com>
 Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
 Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
 Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
 Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
 Satyam Bansal <sbansal1999@gmail.com>
 Sayam Samal <samal.sayam@gmail.com>
 Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
 Shlok Patel <shlokcpatel2001@gmail.com>
 Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
 Steve Howell <showell@zulip.com> <showell30@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@zulipchat.com>
 Steve Howell <showell@zulip.com> <steve@humbughq.com>
 Steve Howell <showell@zulip.com> <steve@zulip.com>
 strifel <info@strifel.de>
 Tim Abbott <tabbott@zulip.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
 Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
 Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
 Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
 Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
 Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
 umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
 umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
 Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
 Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
 Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
 Yash RE <33805964+YashRE42@users.noreply.github.com>
 Yogesh Sirsat <yogeshsirsat56@gmail.com>
 Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
 Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
 Zeeshan Equbal <equbalzeeshan@gmail.com>
 Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
 Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
 Zev Benjamin <zev@zulip.com> <zev@mit.edu>
 Zixuan James Li <p359101898@gmail.com>
 Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>
 Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
 Joseph Ho <josephho678@gmail.com>
 Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
--- a/.npmrc
+++ b/.npmrc
@@ -1 +0,0 @@
 ignore-dep-scripts=true
--- a/.prettierignore
+++ b/.prettierignore
@@ -1,11 +0,0 @@
 pnpm-lock.yaml
 /api_docs/**/*.md
 /corporate/tests/stripe_fixtures
 /help/**/*.md
 /locale
 /templates/**/*.md
 /tools/setup/emoji/emoji_map.json
 /web/third
 /zerver/tests/fixtures
 /zerver/webhooks/*/doc.md
 /zerver/webhooks/*/fixtures
--- a/.pyre_configuration
+++ b/.pyre_configuration
@@ -1,15 +0,0 @@
 {
  "source_directories": ["."],
  "taint_models_path": [
      "stubs/taint",
      "zulip-py3-venv/lib/pyre_check/taint/"
  ],
  "search_path": [
      "stubs/",
      "zulip-py3-venv/lib/pyre_check/stubs/"
  ],
  "typeshed": "zulip-py3-venv/lib/pyre_check/typeshed/",
  "exclude": [
      "/srv/zulip/zulip-py3-venv/.*"
  ]
 }
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -1,15 +0,0 @@
 # https://docs.readthedocs.io/en/stable/config-file/v2.html
 version: 2
 build:
  os: ubuntu-22.04
  tools:
    python: "3.10"
 sphinx:
  configuration: docs/conf.py
  fail_on_warning: true
 python:
  install:
    - requirements: requirements/docs.txt
--- a/.sonarcloud.properties
+++ b/.sonarcloud.properties
@@ -1 +0,0 @@
 sonar.inclusions=**/*.py,**/*.html
--- a/.travis.yml
+++ b/.travis.yml
@@ -0,0 +1,47 @@
 dist: trusty
 before_install:
   - nvm install 0.10
 install:
  - pip install coveralls
  - tools/travis/setup-$TEST_SUITE
  - tools/clean-venv-cache --travis
 cache:
  - apt: false
  - directories:
    - $HOME/phantomjs
    - $HOME/zulip-venv-cache
    - node_modules
    - $HOME/node
 env:
  global:
    - COVERALLS_PARALLEL=true
    - COVERALLS_SERVICE_NAME=travis-pro
    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
    - BOTO_CONFIG=/tmp/nowhere
  matrix:
    - TEST_SUITE=frontend
    - TEST_SUITE=backend
 language: python
 python:
  - "2.7"
  - "3.4"
 matrix:
  include:
    - python: "3.4"
      env: TEST_SUITE=static-analysis
    - python: "2.7"
      env: TEST_SUITE=production
      sudo: required
 # command to run tests
 script:
  - unset GEM_PATH
  - ./tools/travis/$TEST_SUITE
 sudo: required
 services:
 - docker
 addons:
  postgresql: "9.3"
 after_success:
  coveralls
 notifications:
  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN
--- a/.tx/config
+++ b/.tx/config
@@ -1,40 +1,17 @@
 # Migrated from transifex-client format with `tx migrate`
 #
 # See https://developers.transifex.com/docs/using-the-client which hints at
 # this format, but in general, the headings are in the format of:
 #
 # [o:<org>:p:<project>:r:<resource>]
 [main]
 host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
-[o:zulip:p:zulip:r:djangopo]
+[zulip.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
+source_file = static/locale/en/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 file_filter = static/locale/<lang>/LC_MESSAGES/django.po
 lang_map = zh-Hans: zh_CN
-[o:zulip:p:zulip:r:mobile]
+[zulip.translationsjson]
-file_filter = locale/<lang>/mobile.json
+source_file = static/locale/en/translations.json
 source_file = locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 file_filter = static/locale/<lang>/translations.json
 lang_map = zh-Hans: zh-CN
 [o:zulip:p:zulip:r:translationsjson]
 file_filter = locale/<lang>/translations.json
 source_file = locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
 [o:zulip:p:zulip-test:r:djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 [o:zulip:p:zulip-test:r:translationsjson]
 file_filter = locale/<lang>/translations.json
 source_file = locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@@ -1,23 +0,0 @@
 {
    // Recommended VS Code extensions for zulip/zulip.
    //
    // VS Code prompts a user to install the recommended extensions
    // when a workspace is opened for the first time.  The user can
    // also review the list with the 'Extensions: Show Recommended
    // Extensions' command.  See
    // https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions
    // for more information.
    //
    // Extension identifier format: ${publisher}.${name}.
    // Example: vscode.csharp
    "recommendations": [
        "42crunch.vscode-openapi",
        "dbaeumer.vscode-eslint",
        "esbenp.prettier-vscode",
        "ms-vscode-remote.vscode-remote-extensionpack"
    ],
    // Extensions recommended by VS Code which are not recommended for users of zulip/zulip.
    "unwantedRecommendations": []
 }
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -1,172 +0,0 @@
 # Zulip Code of Conduct
 Like the technical community as a whole, the Zulip team and community is
 made up of a mixture of professionals and volunteers from all over the
 world, working on every aspect of the mission, including mentorship,
 teaching, and connecting people.
 Diversity is one of our huge strengths, but it can also lead to
 communication issues and unhappiness. To that end, we have a few ground
 rules that we ask people to adhere to. This code applies equally to
 founders, mentors, and those seeking help and guidance.
 This isn't an exhaustive list of things that you can't do. Rather, take it
 in the spirit in which it's intended --- a guide to make it easier to enrich
 all of us and the technical communities in which we participate.
 ## Expected behavior
 The following behaviors are expected and requested of all community members:
 - Participate. In doing so, you contribute to the health and longevity of
  the community.
 - Exercise consideration and respect in your speech and actions.
 - Attempt collaboration before conflict. Assume good faith.
 - Refrain from demeaning, discriminatory, or harassing behavior and speech.
 - Take action or alert community leaders if you notice a dangerous
  situation, someone in distress, or violations of this code, even if they
  seem inconsequential.
 - Community event venues may be shared with members of the public; be
  respectful to all patrons of these locations.
 ## Unacceptable behavior
 The following behaviors are considered harassment and are unacceptable
 within the Zulip community:
 - Jokes or derogatory language that singles out members of any race,
  ethnicity, culture, national origin, color, immigration status, social and
  economic class, educational level, language proficiency, sex, sexual
  orientation, gender identity and expression, age, size, family status,
  political belief, religion, and mental and physical ability.
 - Violence, threats of violence, or violent language directed against
  another person.
 - Disseminating or threatening to disseminate another person's personal
  information.
 - Personal insults of any sort.
 - Posting or displaying sexually explicit or violent material.
 - Inappropriate photography or recording.
 - Deliberate intimidation, stalking, or following (online or in person).
 - Unwelcome sexual attention. This includes sexualized comments or jokes,
  inappropriate touching or groping, and unwelcomed sexual advances.
 - Sustained disruption of community events, including talks and
  presentations.
 - Advocating for, or encouraging, any of the behaviors above.
 ## Reporting and enforcement
 Harassment and other code of conduct violations reduce the value of the
 community for everyone. If someone makes you or anyone else feel unsafe or
 unwelcome, please report it to the community organizers at
 zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
 report either personally or anonymously.
 If a community member engages in unacceptable behavior, the community
 organizers may take any action they deem appropriate, up to and including a
 temporary ban or permanent expulsion from the community without warning (and
 without refund in the case of a paid event).
 If someone outside the development community (e.g. a user of the Zulip
 software) engages in unacceptable behavior that affects someone in the
 community, we still want to know. Even if we don't have direct control over
 the violator, the community organizers can still support the people
 affected, reduce the chance of a similar violation in the future, and take
 any direct action we can.
 The nature of reporting means it can only help after the fact. If you see
 something you can do while a violation is happening, do it. A lot of the
 harms of harassment and other violations can be mitigated by the victim
 knowing that the other people present are on their side.
 All reports will be kept confidential. In some cases, we may determine that a
 public statement will need to be made. In such cases, the identities of all
 victims and reporters will remain confidential unless those individuals
 instruct us otherwise.
 ## Scope
 We expect all community participants (contributors, paid or otherwise,
 sponsors, and other guests) to abide by this Code of Conduct in all
 community venues, online and in-person, as well as in all private
 communications pertaining to community business.
 This Code of Conduct and its related procedures also applies to unacceptable
 behavior occurring outside the scope of community activities when such
 behavior has the potential to adversely affect the safety and well-being of
 community members.
 ## License and attribution
 This Code of Conduct is adapted from the
 [Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
 under a
 [Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
 license.
 ## Moderating the Zulip community
 Anyone can help moderate the Zulip community by helping make sure that folks are
 aware of the [community guidelines](https://zulip.com/development-community/)
 and this Code of Conduct, and that we maintain a positive and respectful
 atmosphere.
 Here are some guidelines for you how can help:
 - Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
  and just trying to keep the atmosphere warm make the whole community function
  more smoothly. New participants who feel accepted, listened to and respected
  are likely to treat others the same way.
 - Be familiar with the [community
  guidelines](https://zulip.com/development-community/), and cite them liberally
  when a user violates them. Be polite but firm. Some examples:
  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
    you ask a question. As noted in the [guidelines for this
    community](https://zulip.com/development-community/):
    > Use @-mentions sparingly… there is generally no need to @-mention a
    > core contributor unless you need their timely attention.
  - @user, please keep in mind the following [community
    guideline](https://zulip.com/development-community/):
    > Don’t ask the same question in multiple places. Moderators read every
    > public stream, and make sure every question gets a reply.
    I’ve gone ahead and moved the other copy of this message to this thread.
  - If asked a question in a PM that is better discussed in a public stream:
    > Hi @user! Please start by reviewing
    > https://zulip.com/development-community/#community-norms to learn how to
    > get help in this community.
 - Users sometimes think chat.zulip.org is a testing instance. When this happens,
  kindly direct them to use the **#test here** stream.
 - If you see a message that’s posted in the wrong place, go ahead and move it if
  you have permissions to do so, even if you don’t plan to respond to it.
  Leaving the “Send automated notice to new topic” option enabled helps make it
  clear what happened to the person who sent the message.
  If you are responding to a message that's been moved, mention the user in your
  reply, so that the mention serves as a notification of the new location for
  their conversation.
 - If a user is posting spam, please report it to an administrator. They will:
  - Change the user's name to `<name> (spammer)` and deactivate them.
  - Delete any spam messages they posted in public streams.
 - We care very much about maintaining a respectful tone in our community. If you
  see someone being mean or rude, point out that their tone is inappropriate,
  and ask them to communicate their perspective in a respectful way in the
  future. If you don’t feel comfortable doing so yourself, feel free to ask a
  member of Zulip's core team to take care of the situation.
 - Try to assume the best intentions from others (given the range of
  possibilities presented by their visible behavior), and stick with a friendly
  and positive tone even when someone‘s behavior is poor or disrespectful.
  Everyone has bad days and stressful situations that can result in them
  behaving not their best, and while we should be firm about our community
  rules, we should also enforce them with kindness.
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,390 +0,0 @@
 # Contributing guide
 Welcome to the Zulip community!
 ## Zulip development community
 The primary communication forum for the Zulip community is the Zulip
 server hosted at [chat.zulip.org](https://chat.zulip.org/):
 - **Users** and **administrators** of Zulip organizations stop by to
  ask questions, offer feedback, and participate in product design
  discussions.
 - **Contributors to the project**, including the **core Zulip
  development team**, discuss ongoing and future projects, brainstorm
  ideas, and generally help each other out.
 Everyone is welcome to [sign up](https://chat.zulip.org/) and
 participate — we love hearing from our users! Public streams in the
 community receive thousands of messages a week. We recommend signing
 up using the special invite links for
 [users](https://chat.zulip.org/join/t5crtoe62bpcxyisiyglmtvb/),
 [self-hosters](https://chat.zulip.org/join/wnhv3jzm6afa4raenedanfno/)
 and
 [contributors](https://chat.zulip.org/join/npzwak7vpmaknrhxthna3c7p/)
 to get a curated list of initial stream subscriptions.
 To learn how to get started participating in the community, including [community
 norms](https://zulip.com/development-community/#community-norms) and [where to
 post](https://zulip.com/development-community/#where-do-i-send-my-message),
 check out our [Zulip development community
 guide](https://zulip.com/development-community/). The Zulip community is
 governed by a [code of
 conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
 ## Ways to contribute
 To make a code or documentation contribution, read our
 [step-by-step guide](#your-first-codebase-contribution) to getting
 started with the Zulip codebase. A small sample of the type of work that
 needs doing:
 - Bug squashing and feature development on our Python/Django
  [backend](https://github.com/zulip/zulip), web
  [frontend](https://github.com/zulip/zulip), React Native
  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
  [desktop app](https://github.com/zulip/zulip-desktop).
 - Building out our
  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
 - [Writing an integration](https://zulip.com/api/integrations-overview).
 - Improving our [user](https://zulip.com/help/) or
  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
 - [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
  and manually testing pull requests.
 **Non-code contributions**: Some of the most valuable ways to contribute
 don't require touching the codebase at all. For example, you can:
 - Report issues, including both [feature
  requests](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html)
  and [bug
  reports](https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html).
 - [Give feedback](#user-feedback) if you are evaluating or using Zulip.
 - [Participate
  thoughtfully](https://zulip.readthedocs.io/en/latest/contributing/design-discussions.html)
  in design discussions.
 - [Sponsor Zulip](https://github.com/sponsors/zulip) through the GitHub sponsors program.
 - [Translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
  Zulip into your language.
 - [Stay connected](#stay-connected) with Zulip, and [help others
  find us](#help-others-find-zulip).
 ## Your first codebase contribution
 This section has a step by step guide to starting as a Zulip codebase
 contributor. It's long, but don't worry about doing all the steps perfectly;
 no one gets it right the first time, and there are a lot of people available
 to help.
 - First, make an account on the
  [Zulip community server](https://zulip.com/development-community/),
  paying special attention to the
  [community norms](https://zulip.com/development-community/#community-norms).
  If you'd like, introduce yourself in
  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
  your name as the topic. Bonus: tell us about your first impressions of
  Zulip, and anything that felt confusing/broken or interesting/helpful as you
  started using the product.
 - Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
 - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
  getting help in
  [#provision help](https://chat.zulip.org/#narrow/stream/21-provision-help)
  if you run into any troubles.
 - Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
 - Go through the [new application feature
  tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
  how the Zulip codebase is organized and how to find code in it.
 - Read the [Zulip guide to
  Git](https://zulip.readthedocs.io/en/latest/git/index.html) if you
  are unfamiliar with Git or Zulip's rebase-based Git workflow,
  getting help in [#git
  help](https://chat.zulip.org/#narrow/stream/44-git-help) if you run
  into any troubles. Even Git experts should read the [Zulip-specific
  Git tools
  page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
 ### Where to look for an issue
 Now you're ready to pick your first issue! Zulip has several repositories you
 can check out, depending on your interests. There are hundreds of open issues in
 the [main Zulip server and web app
 repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 alone.
 You can look through issues tagged with the "help wanted" label, which is used
 to indicate the issues that are ready for contributions. Some repositories also
 use the "good first issue" label to tag issues that are especially approachable
 for new contributors.
 - [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 - [Mobile apps](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 - [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 - [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
 - [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 ### Picking an issue to work on
 There's a lot to learn while making your first pull request, so start small!
 Many first contributions have fewer than 10 lines of changes (not counting
 changes to tests).
 We recommend the following process for finding an issue to work on:
 1. Read the description of an issue tagged with the "help wanted" label and make
   sure you understand it.
 2. If it seems promising, poke around the product
   (on [chat.zulip.org](https://chat.zulip.org) or in the development
   environment) until you know how the piece being
   described fits into the bigger picture. If after some exploration the
   description seems confusing or ambiguous, post a question on the GitHub
   issue, as others may benefit from the clarification as well.
 3. When you find an issue you like, try to get started working on it. See if you
   can find the part of the code you'll need to modify (`git grep` is your
   friend!) and get some idea of how you'll approach the problem.
 4. If you feel lost, that's OK! Go through these steps again with another issue.
   There's plenty to work on, and the exploration you do will help you learn
   more about the project.
 Note that you are _not_ claiming an issue while you are iterating through steps
 1-4. _Before you claim an issue_, you should be confident that you will be able to
 tackle it effectively.
 Additional tips for the [main server and web app
 repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
 - We especially recommend browsing recently opened issues, as there are more
  likely to be easy ones for you to find.
 - All issues are partitioned into areas like
  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
  through our [list of labels](https://github.com/zulip/zulip/labels), and
  click on some of the `area:` labels to see all the issues related to your
  areas of interest.
 - Avoid issues with the "difficult" label unless you
  understand why it is difficult and are highly confident you can resolve the
  issue correctly and completely.
 ### Claiming an issue
 #### In the main server/web app repository and Zulip Terminal repository
 The Zulip server/web app repository
 ([`zulip/zulip`](https://github.com/zulip/zulip/)) and the Zulip Terminal
 repository ([`zulip/zulip-terminal`](https://github.com/zulip/zulip-terminal/))
 are set up with a GitHub workflow bot called
 [Zulipbot](https://github.com/zulip/zulipbot), which manages issues and pull
 requests in order to create a better workflow for Zulip contributors.
 To claim an issue in these repositories, simply post a comment that says
 `@zulipbot claim` to the issue thread. If the issue is tagged with a [help
 wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 label, Zulipbot will immediately assign the issue to you.
 Note that new contributors can only claim one issue until their first pull request is
 merged. This is to encourage folks to finish ongoing work before starting
 something new. If you would like to pick up a new issue while waiting for review
 on an almost-ready pull request, you can post a comment to this effect on the
 issue you're interested in.
 #### In other Zulip repositories
 There is no bot for other Zulip repositories
 ([`zulip/zulip-mobile`](https://github.com/zulip/zulip-mobile/), etc.). If
 you are interested in claiming an issue in one of these repositories, simply
 post a comment on the issue thread saying that you'd like to work on it. There
 is no need to @-mention the issue creator in your comment.
 Please follow the same guidelines as described above: find an issue labeled
 "help wanted", and only pick up one issue at a time to start with.
 ### Working on an issue
 You're encouraged to ask questions on how to best implement or debug your
 changes -- the Zulip maintainers are excited to answer questions to help you
 stay unblocked and working efficiently. You can ask questions in the [Zulip
 development community](https://zulip.com/development-community/), or on the
 GitHub issue or pull request.
 To get early feedback on any UI changes, we encourage you to post screenshots of
 your work in the [#design
 stream](https://chat.zulip.org/#narrow/stream/101-design) in the [Zulip
 development community](https://zulip.com/development-community/)
 For more advice, see [What makes a great Zulip
 contributor?](#what-makes-a-great-zulip-contributor) below. It's OK if your
 first issue takes you a while; that's normal! You'll be able to work a lot
 faster as you build experience.
 ### Submitting a pull request
 See the [pull request review
 process](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
 guide for detailed instructions on how to submit a pull request, and information
 on the stages of review your PR will go through.
 ### Beyond the first issue
 To find a second issue to work on, we recommend looking through issues with the same
 `area:` label as the last issue you resolved. You'll be able to reuse the
 work you did learning how that part of the codebase works. Also, the path to
 becoming a core developer often involves taking ownership of one of these area
 labels.
 ### Common questions
 - **What if somebody is already working on the issue I want do claim?** There
  are lots of issue to work on! If somebody else is actively working on the
  issue, you can find a different one, or help with
  reviewing their work.
 - **What if somebody else claims an issue while I'm figuring out whether or not to
  work on it?** No worries! You can contribute by providing feedback on
  their pull request. If you've made good progress in understanding part of the
  codebase, you can also find another "help wanted" issue in the same area to
  work on.
 - **What if there is already a pull request for the issue I want to work on?**
  Start by reviewing the existing work. If you agree with the approach, you can
  use the existing pull request (PR) as a starting point for your contribution. If
  you think a different approach is needed, you can post a new PR, with a comment that clearly
  explains _why_ you decided to start from scratch.
 - **Can I come up with my own feature idea and work on it?** We welcome
  suggestions of features or other improvements that you feel would be valuable. If you
  have a new feature you'd like to add, you can start a conversation [in our
  development community](https://zulip.com/development-community/#where-do-i-send-my-message)
  explaining the feature idea and the problem that you're hoping to solve.
 - **I'm waiting for the next round of review on my PR. Can I pick up
  another issue in the meantime?** Someone's first Zulip PR often
  requires quite a bit of iteration, so please [make sure your pull
  request is reviewable][reviewable-pull-requests] and go through at
  least one round of feedback from others before picking up a second
  issue. After that, sure! If
  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
  claim an issue, you can post a comment describing the status of your
  other work on the issue you're interested in, and asking for the
  issue to be assigned to you. Note that addressing feedback on
  in-progress PRs should always take priority over starting a new PR.
 - **I think my PR is done, but it hasn't been merged yet. What's going on?**
  1. **Double-check that you have addressed all the feedback**, including any comments
     on [Git commit
     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
  2. If all the feedback has been addressed, did you [leave a
     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
     explaining that you have done so and **requesting another review**? If not,
     it may not be clear to project maintainers or reviewers that your PR is
     ready for another look.
  3. There may be a pause between initial rounds of review for your PR and final
     review by project maintainers. This is normal, and we encourage you to **work
     on other issues** while you wait.
  4. If you think the PR is ready and haven't seen any updates for a couple
     of weeks, it can be helpful to **leave another comment**. Summarize the
     overall state of the review process and your work, and indicate that you
     are waiting for a review.
  5. Finally, **Zulip project maintainers are people too**! They may be busy
     with other work, and sometimes they might even take a vacation. ;) It can
     occasionally take a few weeks for a PR in the final stages of the review
     process to be merged.
 [reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
 ## What makes a great Zulip contributor?
 Zulip has a lot of experience working with new contributors. In our
 experience, these are the best predictors of success:
 - [Asking great questions][great-questions]. It's very hard to answer a general
  question like, "How do I do this issue?" When asking for help, explain your
  current understanding, including what you've done or tried so far and where
  you got stuck. Post tracebacks or other error messages if appropriate. For
  more advice, check out [our guide][great-questions]!
 - Learning and practicing
  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
 - Submitting carefully tested code. See our [detailed guide on how to review
  code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
  (yours or someone else's).
 - Posting
  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
  for frontend changes.
 - Working to [make your pull requests easy to
  review](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html).
 - Clearly describing what you have implemented and why. For example, if your
  implementation differs from the issue description in some way or is a partial
  step towards the requirements described in the issue, be sure to call
  out those differences.
 - Being responsive to feedback on pull requests. This means incorporating or
  responding to all suggested changes, and leaving a note if you won't be
  able to address things within a few days.
 - Being helpful and friendly on the [Zulip community
  server](https://zulip.com/development-community/).
 [great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
 ## User feedback
 Nearly every feature we develop starts with a user request. If you are part
 of a group that is either using or considering using Zulip, we would love to
 hear about your experience with the product. If you're not sure what to
 write, here are some questions we're always very curious to know the answer
 to:
 - Evaluation: What is the process by which your organization chose or will
  choose a group chat product?
 - Pros and cons: What are the pros and cons of Zulip for your organization,
  and the pros and cons of other products you are evaluating?
 - Features: What are the features that are most important for your
  organization? In the best-case scenario, what would your chat solution do
  for you?
 - Onboarding: If you remember it, what was your impression during your first
  few minutes of using Zulip? What did you notice, and how did you feel? Was
  there anything that stood out to you as confusing, or broken, or great?
 - Organization: What does your organization do? How big is the organization?
  A link to your organization's website?
 You can contact us in the [#feedback stream of the Zulip development
 community](https://chat.zulip.org/#narrow/stream/137-feedback) or
 by emailing [support@zulip.com](mailto:support@zulip.com).
 ## Outreach programs
 Zulip regularly participates in [Google Summer of Code
 (GSoC)](https://developers.google.com/open-source/gsoc/) and
 [Outreachy](https://www.outreachy.org/). We have been a GSoC mentoring
 organization since 2016, and we accept 15-20 GSoC participants each summer. In
 the past, we’ve also participated in [Google
 Code-In](https://developers.google.com/open-source/gci/), and hosted summer
 interns from Harvard, MIT, and Stanford.
 Check out our [outreach programs
 overview](https://zulip.readthedocs.io/en/latest/outreach/overview.html) to learn
 more about participating in an outreach program with Zulip. Most of our program
 participants end up sticking around the project long-term, and many have become
 core team members, maintaining important parts of the project. We hope you
 apply!
 ## Stay connected
 Even if you are not logging into the development community on a regular basis,
 you can still stay connected with the project.
 - Follow us [on Twitter](https://twitter.com/zulip).
 - Subscribe to [our blog](https://blog.zulip.org/).
 - Join or follow the project [on LinkedIn](https://www.linkedin.com/company/zulip-project/).
 ## Help others find Zulip
 Here are some ways you can help others find Zulip:
 - Star us on GitHub. There are four main repositories:
  [server/web](https://github.com/zulip/zulip),
  [mobile](https://github.com/zulip/zulip-mobile),
  [desktop](https://github.com/zulip/zulip-desktop), and
  [Python API](https://github.com/zulip/python-zulip-api).
 - "Like" and retweet [our tweets](https://twitter.com/zulip).
 - Upvote and post feedback on Zulip on comparison websites. A couple specific
  ones to highlight:
  - [AlternativeTo](https://alternativeto.net/software/zulip-chat-server/). You can also
    [upvote Zulip](https://alternativeto.net/software/slack/) on their page
    for Slack.
  - [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
    it, and upvote the reasons why people like Zulip that you find most
    compelling.
--- a/15
+++ b/15
@@ -0,0 +1,15 @@
 FROM ubuntu:trusty
 EXPOSE 9991
 RUN apt-get update && apt-get install -y \
  python-pbs \
  wget
 RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER zulip
 RUN ln -nsf /srv/zulip ~/zulip
 WORKDIR /srv/zulip
--- a/25
+++ b/25
@@ -1,25 +0,0 @@
 # This is a multiarch Dockerfile.  See https://docs.docker.com/desktop/multi-arch/
 #
 # To set up the first time:
 #     docker buildx create --name multiarch --use
 #
 # To build:
 #     docker buildx build --platform linux/amd64,linux/arm64 \
 #       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
 # Currently the PostgreSQL images do not support automatic upgrading of
 # the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
 # https://hub.docker.com/r/groonga/pgroonga/tags
 ARG PGROONGA_VERSION=latest
 ARG POSTGRESQL_VERSION=14
 FROM groonga/pgroonga:$PGROONGA_VERSION-alpine-$POSTGRESQL_VERSION-slim
 # Install hunspell, Zulip stop words, and run Zulip database
 # init.
 RUN apk add -U --no-cache hunspell-en
 RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix
 COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/local/share/postgresql/tsearch_data/zulip_english.stop
 COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
 COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql
--- a/18
+++ b/18
@@ -1,18 +0,0 @@
 Copyright 2012–2015 Dropbox, Inc., 2015–2021 Kandra Labs, Inc., and contributors
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this project except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 The software includes some works released by third parties under other
 free and open source licenses. Those works are redistributed under the
 license terms under which the works were received. For more details,
 see the ``docs/THIRDPARTY`` file included with this distribution.
--- a/README.md
+++ b/README.md
@@ -1,83 +1,226 @@
 **[Zulip overview](#zulip-overview)** |
 **[Community](#community)** |
 **[Installing for dev](#installing-the-zulip-development-environment)** |
 **[Installing for production](#running-zulip-in-production)** |
 **[Ways to contribute](#ways-to-contribute)** |
 **[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
 **[License](#license)**
 # Zulip overview
-[Zulip](https://zulip.com) is an open-source team collaboration tool with unique
+Zulip is a powerful, open source group chat application. Written in
-[topic-based threading][why-zulip] that combines the best of email and chat to
+Python and using the Django framework, Zulip supports both private
-make remote work productive and delightful. Fortune 500 companies, [leading open
+messaging and group chats via conversation streams.
 source projects][rust-case-study], and thousands of other organizations use
 Zulip every day. Zulip is the only [modern team chat app][features] that is
 designed for both live and asynchronous conversations.
-Zulip is built by a distributed community of developers from all around the
+Zulip also supports fast search, drag-and-drop file uploads, image
-world, with 74+ people who have each contributed 100+ commits. With
+previews, group private messages, audible notifications,
-over 1000 contributors merging over 500 commits a month, Zulip is the
+missed-message emails, desktop apps, and much more.
 largest and fastest growing open source team chat project.
-Come find us on the [development community chat](https://zulip.com/development-community/)!
+Further information on the Zulip project and its features can be found
 at https://www.zulip.org.
-[![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master)
 [![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff)
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
 [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
 [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
 [![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
 [![GitHub Sponsors](https://img.shields.io/github/sponsors/zulip)](https://github.com/sponsors/zulip)
-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
+## Community
 [why-zulip]: https://zulip.com/why-zulip/
 [rust-case-study]: https://zulip.com/case-studies/rust/
 [features]: https://zulip.com/features/
-## Getting started
+There are several places online where folks discuss Zulip.
- **Contributing code**. Check out our [guide for new
+One of those places is our [public Zulip instance](https://zulip.tabbott.net/).
-  contributors](https://zulip.readthedocs.io/en/latest/contributing/contributing.html)
+You can go through the simple signup process at that link, and then you
-  to get started. We have invested in making Zulip’s code highly
+will soon be talking to core Zulip developers and other users.  To get
-  readable, thoughtfully tested, and easy to modify. Beyond that, we
+help in real time, you will have the best luck finding core developers
-  have written an extraordinary 150K words of documentation for Zulip
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get answered
-  contributors.
+within a day.
- **Contributing non-code**. [Report an
+We have a [Google mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
-  issue](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#reporting-issues),
+that is currently pretty low traffic.  It is where we do things like
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
+announce public meetings or major releases.  You can also use it to
-  Zulip into your language, or [give us
+ask questions about features or possible bugs.
  feedback](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#user-feedback).
  We'd love to hear from you, whether you've been using Zulip for years, or are just
  trying it out for the first time.
- **Checking Zulip out**. The best way to see Zulip in action is to drop by the
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
-  [Zulip community server](https://zulip.com/development-community/). We also
+track Zulip-related issues (and store our code, of course).
-  recommend reading about Zulip's [unique
+Anybody with a Github account should be able to create Issues there
-  approach](https://zulip.com/why-zulip/) to organizing conversations.
+pertaining to bugs or enhancement requests.  We also use Pull
 Requests as our primary mechanism to receive code contributions.
- **Running a Zulip server**. Self-host Zulip directly on Ubuntu or Debian
+## Installing the Zulip Development environment
  Linux, in [Docker](https://github.com/zulip/docker-zulip), or with prebuilt
  images for [Digital Ocean](https://marketplace.digitalocean.com/apps/zulip) and
  [Render](https://render.com/docs/deploy-zulip).
  Learn more about [self-hosting Zulip](https://zulip.com/self-hosting/).
- **Using Zulip without setting up a server**. Learn about [Zulip
+The Zulip development environment is the recommended option for folks
-  Cloud](https://zulip.com/plans/) hosting options. Zulip sponsors free [Zulip
+interested in trying out Zulip.  This is documented in [the developer
-  Cloud Standard](https://zulip.com/plans/) for hundreds of worthy
+installation guide][dev-install].
  organizations, including [fellow open-source
  projects](https://zulip.com/for/open-source/).
- **Participating in [outreach
+## Running Zulip in production
  programs](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#outreach-programs)**
  like [Google Summer of Code](https://developers.google.com/open-source/gsoc/)
  and [Outreachy](https://www.outreachy.org/).
- **Supporting Zulip**. Advocate for your organization to use Zulip, become a
+Zulip in production only supports Ubuntu 14.04 right now, but work is
-  [sponsor](https://github.com/sponsors/zulip), write a review in the mobile app
+ongoing on adding support for additional platforms. The installation
-  stores, or [help others find
+process is documented at https://zulip.org/server.html and in more
-  Zulip](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#help-others-find-zulip).
+detail in [the
 documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
-You may also be interested in reading our [blog](https://blog.zulip.org/), and
+## Ways to contribute
 following us on [Twitter](https://twitter.com/zulip) and
 [LinkedIn](https://www.linkedin.com/company/zulip-project/).
-Zulip is distributed under the
+Zulip welcomes all forms of contributions!  The page documents the
-[Apache 2.0](https://github.com/zulip/zulip/blob/main/LICENSE) license.
+Zulip development process.
 * **Pull requests**. Before a pull request can be merged, you need to
 to sign the [Dropbox Contributor License Agreement][cla].  Also,
 please skim our [commit message style guidelines][doc-commit-style].
 * **Testing**. The Zulip automated tests all run automatically when
 you submit a pull request, but you can also run them all in your
 development environment following the instructions in the [testing
 docs][doc-test]. You can also try out [our new desktop
 client][electron], which is in alpha; we'd appreciate testing and
 [feedback](https://github.com/zulip/zulip-electron/issues/new).
 * **Developer Documentation**.  Zulip has a growing collection of
 developer documentation on [Read The Docs][doc].  Recommended reading
 for new contributors includes the [directory structure][doc-dirstruct]
 and [new feature tutorial][doc-newfeat]. You can also improve
 [Zulip.org][z-org].
 * **Mailing lists and bug tracker**. Zulip has a [development
 discussion mailing list][gg-devel] and uses [GitHub issues
 ][gh-issues].  There are also lists for the [Android][email-android]
 and [iOS][email-ios] apps.  Feel free to send any questions or
 suggestions of areas where you'd love to see more documentation to the
 relevant list!  Please report any security issues you discover to
 zulip-security@googlegroups.com.
 * **App codebases**. This repository is for the Zulip server and web
 app (including most integrations); the [desktop][], [Android][], and
 [iOS][] apps, are separate repositories, as are our [experimental
 React Native iOS app][ios-exp] and [alpha Electron desktop
 app][electron].
 * **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
 integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
 and [Trello][]), plus [node.js API bindings][node], and a [full-text search
 PostgreSQL extension][tsearch], as separate repos.
 * **Translations**.  Zulip is in the process of being translated into
 10+ languages, and we love contributions to our translations.  See our
 [translating documentation][transifex] if you're interested in
 contributing!
 [cla]: https://opensource.dropbox.com/cla/
 [dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
 [doc]: https://zulip.readthedocs.io/
 [doc-commit-style]: http://zulip.readthedocs.io/en/latest/code-style.html#commit-messages
 [doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
 [doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
 [doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
 [electron]: https://github.com/zulip/zulip-electron
 [gg-devel]: https://groups.google.com/forum/#!forum/zulip-devel
 [gh-issues]: https://github.com/zulip/zulip/issues
 [desktop]: https://github.com/zulip/zulip-desktop
 [android]: https://github.com/zulip/zulip-android
 [ios]: https://github.com/zulip/zulip-ios
 [ios-exp]: https://github.com/zulip/zulip-mobile
 [email-android]: https://groups.google.com/forum/#!forum/zulip-android
 [email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
 [hubot-adapter]: https://github.com/zulip/hubot-zulip
 [jenkins]: https://github.com/zulip/zulip-jenkins-plugin
 [node]: https://github.com/zulip/zulip-node
 [phab]: https://github.com/zulip/phabricator-to-zulip
 [puppet]: https://github.com/matthewbarr/puppet-zulip
 [redmine]: https://github.com/zulip/zulip-redmine-plugin
 [trello]: https://github.com/zulip/trello-to-zulip
 [tsearch]: https://github.com/zulip/tsearch_extras
 [transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
 [z-org]: https://github.com/zulip/zulip.github.io
 ## How to get involved with contributing to Zulip
 First, subscribe to the Zulip [development discussion mailing
 list][gg-devel].
 The Zulip project uses a system of labels in our [issue
 tracker][gh-issues] to make it easy to find a project if you don't
 have your own project idea in mind or want to get some experience with
 working on Zulip before embarking on a larger project you have in
 mind:
 * [Integrations](https://github.com/zulip/zulip/labels/integrations).
  Integrate Zulip with another piece of software and contribute it
  back to the community!  Writing an integration can be a great first
  contribution.  There's detailed documentation on how to write
  integrations in [the Zulip integration writing
  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
 * [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
  Smaller projects that might be a great first contribution.
 * [Documentation](https://github.com/zulip/zulip/labels/documentation):
  The Zulip project loves contributions of new documentation.
 * [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
  A broader list of projects that nobody is currently working on.
 * [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
  These are open issues about making it possible to install Zulip on a
  wider range of platforms.
 * [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
 * [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
  Browsing this list can be a great way to find feature ideas to
  implement that other Zulip users are excited about.
 * [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html): The
  projects that are [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).  These are great projects if you're looking to make an impact.
 If you're excited about helping with an open issue, just post on the
 conversation thread that you're working on it.  You're encouraged to
 ask questions on how to best implement or debug your changes -- the
 Zulip maintainers are excited to answer questions to help you stay
 unblocked and working efficiently.
 We also welcome suggestions of features that you feel would be
 valuable or changes that you feel would make Zulip a better open
 source project, and are happy to support you in adding new features or
 other user experience improvements to Zulip.
 If you have a new feature you'd like to add, we recommend you start by
 opening a GitHub issue about the feature idea explaining the problem
 that you're hoping to solve and that you're excited to work on it.  A
 Zulip maintainer will usually reply within a day with feedback on the
 idea, notes on any important issues or concerns, and and often tips on
 how to implement or test it.  Please feel free to ping the thread if
 you don't hear a response from the maintainers -- we try to be very
 responsive so this usually means we missed your message.
 For significant changes to the visual design, user experience, data
 model, or architecture, we highly recommend posting a mockup,
 screenshot, or description of what you have in mind to zulip-devel@ to
 get broad feedback before you spend too much time on implementation
 details.
 Finally, before implementing a larger feature, we highly recommend
 looking at the new feature tutorial and coding style guidelines on
 ReadTheDocs.
 Feedback on how to make this development process more efficient, fun,
 and friendly to new contributors is very welcome!  Just send an email
 to the Zulip Developers list with your thoughts.
 ## License
 Copyright 2011-2015 Dropbox, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 The software includes some works released by third parties under other
 free and open source licenses. Those works are redistributed under the
 license terms under which the works were received. For more details,
 see the ``THIRDPARTY`` file included with this distribution.
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -1,37 +0,0 @@
 # Security policy
 ## Reporting a vulnerability
 We love responsible reports of (potential) security issues in Zulip,
 whether in the latest release or our development branch.
 Our security contact is security@zulip.com. Reporters should expect a
 response within 24 hours.
 Please include details on the issue and how you'd like to be credited
 in our release notes when we publish the fix.
 Our [security model][security-model] document may be a helpful
 resource.
 ## Security announcements
 We send security announcements to our [announcement mailing
 list](https://groups.google.com/g/zulip-announce). If you are running
 Zulip in production, you should subscribe, by clicking "Join group" at
 the top of that page.
 ## Supported versions
 Zulip provides security support for the latest major release, in the
 form of minor security/maintenance releases.
 We work hard to make [upgrades][upgrades] reliable, so that there's no
 reason to run older major releases.
 See also our documentation on the [Zulip release
 lifecycle][release-lifecycle].
 [security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
 [upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
 [release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html
--- a/567
+++ b/567
@@ -0,0 +1,567 @@
 Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: Zulip
 Upstream-Contact: Zulip Development Discussion <zulip-devel@googlegroups.com>
 Source: https://zulip.org/
 Comment:
 Unless otherwise noted, the Zulip software is distributed under the Apache
 License, Version 2.0. The software includes some works released by third
 parties under other free and open source licenses. Those works are
 redistributed under the license terms under which the works were received.
 .
 While Dropbox has sought to provide complete and accurate licensing
 information for each FOSS package, Dropbox does not represent or warrant
 that the licensing information provided herein is correct or error-free.
 Recipients of the Zulip software should investigate the identified FOSS
 packages to confirm the accuracy of the licensing information provided.
 Recipients are also encouraged to notify Dropbox of any inaccurate
 information or errors found in these notices.
 Files: *
 Copyright: 2011-2015 Dropbox, Inc.
 License: Apache-2
 Files: api/*
 Copyright: 2012-2014 Dropbox, Inc
 License: Expat
 Files: api/integrations/perforce/git_p4.py
 Copyright: 2007 Simon Hausmann <simon@lst.de>,
 2007 Trolltech ASA
 License: Expat
 Comment: https://raw.github.com/git/git/34022ba/git-p4.py
 Files: bots/jabber_mirror_backend.py
 Copyright: 2013 Permabit, Inc., 2013-2014 Dropbox, Inc.
 License: Expat
 Files: confirmation/*
 Copyright: 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 License: BSD-3-Clause
 Files: puppet/apt/*
 Copyright: 2011, Evolving Web Inc.
 License: Expat
 Files: puppet/stdlib/*
 Copyright: 2011, Krzysztof Wilczynski
 2011, Puppet Labs Inc
 License: Apache-2.0
 File: puppet/zulip_internal/files/mediawiki/Auth_remoteuser.php
 Copyright: 2006 Otheus Shelling
 2007 Rusty Burchfield
 2009 James Kinsman
 2010 Daniel Thomas
 2010 Ian Ward Comfort
 License: GPL-2.0
 Comment: Not linked.
 Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
 Copyright: 2005 Francesc Guasch
 License: GPL-2.0
 Comment: Not linked.
 Files: puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_postgres.pl
 Copyright: 2007-2015 Greg Sabino Mullane
 License: BSD-2-Clause
 Files: puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh
 Copyright: 2011 Chris Freeman
 License: GPL-2.0
 Files: puppet/zulip_internal/files/trac/cgi-bin/
 Copyright: 2003-2009 Edgewall Software
 2003-2004 Jonas Borgström <jonas@edgewall.com>
 License: BSD-3-Clause
 Files: puppet/zulip_internal/files/pagerduty_nagios.pl
 Copyright: 2011, PagerDuty, Inc.
 License: BSD-3-Clause
 Files: puppet/zulip_internal/files/zulip-ec2-configure-interfaces
 Copyright: 2013, Dropbox, Inc.
 License: Expat
 Files: static/audio/zulip.*
 Copyright: 2011 Vidsyn
 License: CC-0-1.0
 Files: static/styles/thirdparty-fonts.css
 Copyright: 2012-2013 Dave Gandy
 License: Expat
 Files: static/third/fontawesome/*
 Copyright: 2012-2013 Dave Gandy
 License: SIL-OFL-1.1
 Files: static/third/bootstrap/bootstrap-btn.css
 Copyright: 2011-2014 Twitter, Inc
 License: Expat
 Files: static/third/bootstrap/css/bootstrap-responsive.css static/third/bootstrap/css/bootstrap.css
 Copyright: 2012 Twitter, Inc
 License: Apache-2.0
 Comment: The software has been modified.
 Files: static/third/bootstrap/js/bootstrap.js
 Copyright: 2012 Twitter, Inc
 License: Apache-2.0
 Comment: The software has been modified.
 Files: static/third/bootstrap-notify/*
 Copyright: 2013 Nijiko Yonskai
 2012 Goodybag, Inc.
 2012 Twitter, Inc
 License: Apache-2.0
 Comment: The software has been modified.
 Files: static/third/gemoji/images/emoji/unicode/* tools/setup/emoji_dump/*.ttf
 Copyright: Google, Inc.
 License: Apache-2.0
 Comment: These are actually Noto Emoji, not gemoji.
 Files: static/third/html5-formdata/formdata.js
 Copyright: 2010 François de Metz
 License: Expat
 Comment: See https://github.com/francois2metz/html5-formdata
 Files: src/zulip/static/third/jquery/*
 Copyright: 2011, John Resig
 2011, The Dojo Foundation
 License: Expat
 Files: static/third/jquery-autosize/jquery.autosize.js
 Copyright: 2013 Jack Moore
 License: Expat
 Files: static/third/jquery-caret/*
 Copyright: 2012, 2013 Andrew C. Dvorak
 License: Expat
 Files: static/third/jquery-filedrop/jquery.filedrop.js
 Copyright: Resopollution
 License: Expat
 Files: static/third/jquery-form/jquery.form.js
 Copyright: M. Alsup
 License: Expat or GPL-2.0
 Files: static/third/jquery-idle/jquery.idle.js
 Copyright: 2011-2013 Henrique Boaventura
 License: Expat
 Comment: The software has been modified.
 Files: static/third/jquery-mousewheel/jquery.mousewheel.js
 Copyright: 2011 Brandon Aaron
 License: Expat
 Files: static/third/jquery-perfect-scrollbar/*
 Copyright: 2012 HyeonJe Jun
 License: Expat
 Files: static/third/jquery-throttle-debounce/*
 Copyright: 2010 "Cowboy" Ben Alman
 License: Expat or GPL
 Files: static/third/jquery-validate/*
 Copyright: 2006 - 2011 Jörn Zaefferer
 License: Expat
 Files: src/zulip/static/third/lazyload/*
 Copyright: 2011 Ryan Grove
 License: Expat
 Files: static/third/marked/*
 Copyright: 2011-2013, Christopher Jeffrey
 License: Expat
 Files: static/third/string-prototype-codepointat/*
 Copyright: 2014 Mathias Bynens
 License: Expat
 Files: static/third/sockjs/sockjs-0.3.4.js
 Copyright: 2011-2012 VMware, Inc.
 2012 Douglas Crockford
 License: Expat and public-domain
 Files: static/third/sorttable/sorttable.js
 Copyright: 2007 Stuart Langridge
 License: X11
 Files: static/third/sourcesans/*
 Copyright: 2010, 2012, 2014 Adobe Systems Incorporated
 License: SIL-OFL-1.1
 Files: static/third/spectrum/*
 Copyright: 2013 Brian Grinstead
 License: Expat
 Files: static/third/spin/spin.js
 Copyright: 2011-2013 Felix Gnass
 License: Expat
 Files: static/third/underscore/underscore.js
 Copyright: 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
 License: Expat
 Comment: https://github.com/jashkenas/underscore/blob/master/LICENSE
 Files: static/third/winchan/*
 Copyright: 2012 Lloyd Hilaiel
 License: Expat
 Comment: https://github.com/mozilla/winchan
 Files: static/third/xdate/*
 Copyright: 2010 C. F., Wong
 License: Expat
 Files: static/third/zocial/*
 Copyright: Sam Collins
 License: Expat
 Files: tools/inject-messages/othello
 Copyright: Shakespeare
 License: public-domain
 Files: tools/jslint/jslint.js
 Copyright: 2002 Douglas Crockford
 License: XXX-good-not-evil
 Files: tools/review
 Copyright: 2010 Ksplice, Inc.
 License: Apache-2.0
 Files: zerver/lib/bugdown/codehilite.py zerver/lib/bugdown/fenced_code.py
 Copyright: 2006-2008 Waylan Limberg
 License: BSD-3-Clause
 Comment: https://pypi.python.org/pypi/Markdown
 Files: zerver/lib/ccache.py
 Copyright: 2013 David Benjamin and Alan Huang
 License: Expat
 Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/setup/generate_secrets.py
 Copyright: Django Software Foundation and individual contributors
 License: BSD-3-Clause
 Files: frontend_tests/casperjs/*
 Copyright: 2011-2012 Nicolas Perriault
 Joyent, Inc. and other Node contributors
 License: Expat
 Files: frontend_tests/casperjs/modules/vendors/*
 Copyright: 2011, Jeremy Ashkenas
 License: Expat
 License: Apache-2.0
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 .
     http://www.apache.org/licenses/LICENSE-2.0
 .
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 .
 On Debian systems, the full text of the Apache License version 2 can
 be found in /usr/share/common-licenses/Apache-2.0.
 License: BSD-2-clause
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
 1. Redistributions of source code must retain the above copyright
    notice(s), this list of conditions and the following disclaimer
    unmodified other than the allowable addition of one or more
    copyright notices.
 2. Redistributions in binary form must reproduce the above copyright
    notice(s), this list of conditions and the following disclaimer in
    the documentation and/or other materials provided with the
    distribution.
 .
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 License: BSD-3-Clause
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
 are met:
 .
 1. Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.
 .
 2. Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the distribution.
 .
 3. Neither the name of the copyright holder nor the names of its
    contributors may be used to endorse or promote products derived from
    this software without specific prior written permission.
 .
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 License: CC-0-1.0
 Creative Commons CC0 1.0 Universal
 CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
 LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
 ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION
 ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE
 USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND
 DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT
 OR THE INFORMATION OR WORKS PROVIDED HEREUNDER.
 .
 Statement of Purpose
 .
 The laws of most jurisdictions throughout the world automatically confer
 exclusive Copyright and Related Rights (defined below) upon the creator
 and subsequent owner(s) (each and all, an "owner") of an original work
 of authorship and/or a database (each, a "Work").
 .
 Certain owners wish to permanently relinquish those rights to a Work for
 the purpose of contributing to a commons of creative, cultural and
 scientific works ("Commons") that the public can reliably and without
 fear of later claims of infringement build upon, modify, incorporate in
 other works, reuse and redistribute as freely as possible in any form
 whatsoever and for any purposes, including without limitation commercial
 purposes. These owners may contribute to the Commons to promote the
 ideal of a free culture and the further production of creative, cultural
 and scientific works, or to gain reputation or greater distribution for
 their Work in part through the use and efforts of others.
 .
 For these and/or other purposes and motivations, and without any
 expectation of additional consideration or compensation, the person
 associating CC0 with a Work (the "Affirmer"), to the extent that he or
 she is an owner of Copyright and Related Rights in the Work, voluntarily
 elects to apply CC0 to the Work and publicly distribute the Work under
 its terms, with knowledge of his or her Copyright and Related Rights in
 the Work and the meaning and intended legal effect of CC0 on those
 rights.
 .
 1. Copyright and Related Rights. A Work made available under CC0 may be
 protected by copyright and related or neighboring rights ("Copyright and
 Related Rights"). Copyright and Related Rights include, but are not
 limited to, the following:
 .
 i. the right to reproduce, adapt, distribute, perform, display,
 communicate, and translate a Work;
 .
 ii. moral rights retained by the original author(s) and/or performer(s);
 .
 iii. publicity and privacy rights pertaining to a person's image or
 likeness depicted in a Work;
 .
 iv. rights protecting against unfair competition in regards to a Work,
 subject to the limitations in paragraph 4(a), below;
 .
 v. rights protecting the extraction, dissemination, use and reuse of
 data in a Work;
 .
 vi. database rights (such as those arising under Directive 96/9/EC of
 the European Parliament and of the Council of 11 March 1996 on the legal
 protection of databases, and under any national implementation thereof,
 including any amended or successor version of such directive); and
 .
 vii. other similar, equivalent or corresponding rights throughout the
 world based on applicable law or treaty, and any national
 implementations thereof.
 .
 2. Waiver. To the greatest extent permitted by, but not in contravention
 of, applicable law, Affirmer hereby overtly, fully, permanently,
 irrevocably and unconditionally waives, abandons, and surrenders all of
 Affirmer's Copyright and Related Rights and associated claims and causes
 of action, whether now known or unknown (including existing as well as
 future claims and causes of action), in the Work (i) in all territories
 worldwide, (ii) for the maximum duration provided by applicable law or
 treaty (including future time extensions), (iii) in any current or
 future medium and for any number of copies, and (iv) for any purpose
 whatsoever, including without limitation commercial, advertising or
 promotional purposes (the "Waiver"). Affirmer makes the Waiver for the
 benefit of each member of the public at large and to the detriment of
 Affirmer's heirs and successors, fully intending that such Waiver shall
 not be subject to revocation, rescission, cancellation, termination, or
 any other legal or equitable action to disrupt the quiet enjoyment of
 the Work by the public as contemplated by Affirmer's express Statement
 of Purpose.
 .
 3. Public License Fallback. Should any part of the Waiver for any reason
 be judged legally invalid or ineffective under applicable law, then the
 Waiver shall be preserved to the maximum extent permitted taking into
 account Affirmer's express Statement of Purpose. In addition, to the
 extent the Waiver is so judged Affirmer hereby grants to each affected
 person a royalty-free, non transferable, non sublicensable, non
 exclusive, irrevocable and unconditional license to exercise Affirmer's
 Copyright and Related Rights in the Work (i) in all territories
 worldwide, (ii) for the maximum duration provided by applicable law or
 treaty (including future time extensions), (iii) in any current or
 future medium and for any number of copies, and (iv) for any purpose
 whatsoever, including without limitation commercial, advertising or
 promotional purposes (the "License"). The License shall be deemed
 effective as of the date CC0 was applied by Affirmer to the Work. Should
 any part of the License for any reason be judged legally invalid or
 ineffective under applicable law, such partial invalidity or
 ineffectiveness shall not invalidate the remainder of the License, and
 in such case Affirmer hereby affirms that he or she will not (i)
 exercise any of his or her remaining Copyright and Related Rights in the
 Work or (ii) assert any associated claims and causes of action with
 respect to the Work, in either case contrary to Affirmer's express
 Statement of Purpose.
 .
 4. Limitations and Disclaimers.
 .
 a. No trademark or patent rights held by Affirmer are waived, abandoned,
 surrendered, licensed or otherwise affected by this document.
 .
 b. Affirmer offers the Work as-is and makes no representations or
 warranties of any kind concerning the Work, express, implied, statutory
 or otherwise, including without limitation warranties of title,
 merchantability, fitness for a particular purpose, non infringement, or
 the absence of latent or other defects, accuracy, or the present or
 absence of errors, whether or not discoverable, all to the greatest
 extent permissible under applicable law.
 .
 c. Affirmer disclaims responsibility for clearing rights of other
 persons that may apply to the Work or any use thereof, including without
 limitation any person's Copyright and Related Rights in the Work.
 Further, Affirmer disclaims responsibility for obtaining any necessary
 consents, permissions or other rights required for any use of the Work.
 .
 d. Affirmer understands and acknowledges that Creative Commons is not a
 party to this document and has no duty or obligation with respect to
 this CC0 or use of the Work.
 License: Expat
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 .
 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.
 .
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 License: GPL-2.0
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; version 2, dated June, 1991.
 .
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 .
 On Debian systems, the complete text of the GNU General Public License
 can be found in /usr/share/common-licenses/GPL-2 file.
 License: SIL-OFL-1.1
 ---------------------------------------------------------------------------
 SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
 ---------------------------------------------------------------------------
 .
 PREAMBLE
 .
 The goals of the Open Font License (OFL) are to stimulate worldwide development
 of collaborative font projects, to support the font creation efforts of academic
 and linguistic communities, and to provide a free and open framework in which
 fonts may be shared and improved in partnership with others.
 .
 The OFL allows the licensed fonts to be used, studied, modified and redistributed
 freely as long as they are not sold by themselves. The fonts, including any
 derivative works, can be bundled, embedded, redistributed and/or sold with any
 software provided that any reserved names are not used by derivative works. The
 fonts and derivatives, however, cannot be released under any other type of license.
 The requirement for fonts to remain under this license does not apply to any
 document created using the fonts or their derivatives.
 .
 DEFINITIONS
 .
 "Font Software" refers to the set of files released by the Copyright Holder(s) under
 this license and clearly marked as such. This may include source files, build
 scripts and documentation.
 .
 "Reserved Font Name" refers to any names specified as such after the copyright
 statement(s).
 .
 "Original Version" refers to the collection of Font Software components as
 distributed by the Copyright Holder(s).
 .
 "Modified Version" refers to any derivative made by adding to, deleting, or
 substituting -- in part or in whole -- any of the components of the Original Version,
 by changing formats or by porting the Font Software to a new environment.
 .
 "Author" refers to any designer, engineer, programmer, technical writer or other
 person who contributed to the Font Software.
 .
 PERMISSION & CONDITIONS
 .
 Permission is hereby granted, free of charge, to any person obtaining a copy of the
 Font Software, to use, study, copy, merge, embed, modify, redistribute, and sell
 modified and unmodified copies of the Font Software, subject to the following
 conditions:
 .
 1) Neither the Font Software nor any of its individual components, in Original or
 Modified Versions, may be sold by itself.
 .
 2) Original or Modified Versions of the Font Software may be bundled, redistributed
 and/or sold with any software, provided that each copy contains the above copyright
 notice and this license. These can be included either as stand-alone text files,
 human-readable headers or in the appropriate machine-readable metadata fields within
 text or binary files as long as those fields can be easily viewed by the user.
 .
 3) No Modified Version of the Font Software may use the Reserved Font Name(s) unless
 explicit written permission is granted by the corresponding Copyright Holder. This
 restriction only applies to the primary font name as presented to the users.
 .
 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font Software shall
 not be used to promote, endorse or advertise any Modified Version, except to
 acknowledge the contribution(s) of the Copyright Holder(s) and the Author(s) or with
 their explicit written permission.
 .
 5) The Font Software, modified or unmodified, in part or in whole, must be distributed
 entirely under this license, and must not be distributed under any other license. The
 requirement for fonts to remain under this license does not apply to any document
 created using the Font Software.
 .
 TERMINATION
 .
 This license becomes null and void if any of the above conditions are not met.
 .
 DISCLAIMER
 .
 THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER
 RIGHT. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR
 INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE.
--- a/110
+++ b/110
@@ -1,108 +1,84 @@
 # -*- mode: ruby -*-
-Vagrant.require_version ">= 2.2.6"
+VAGRANTFILE_API_VERSION = "2"
 def command?(name)
  `which #{name} > /dev/null 2>&1`
  $?.success?
 end
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  # For LXC. VirtualBox hosts use a different box, described below.
  config.vm.box = "fgrehm/trusty64-lxc"
 Vagrant.configure("2") do |config|
  # The Zulip development environment runs on 9991 on the guest.
  host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
+  http_proxy = https_proxy = no_proxy = ""
  host_ip_addr = "127.0.0.1"
  # System settings for the virtual machine.
  vm_num_cpus = "2"
  vm_memory = "2048"
  ubuntu_mirror = ""
  vboxadd_version = nil
  config.vm.box = "bento/ubuntu-20.04"
  config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder ".", "/srv/zulip", docker_consistency: "z"
+  config.vm.synced_folder ".", "/srv/zulip"
-  vagrant_config_file = ENV["HOME"] + "/.zulip-vagrant-config"
+  vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
  if File.file?(vagrant_config_file)
    IO.foreach(vagrant_config_file) do |line|
      line.chomp!
      key, value = line.split(nil, 2)
      case key
-      when /^([#;]|$)/ # ignore comments
+      when /^([#;]|$)/; # ignore comments
      when "HTTP_PROXY"; http_proxy = value
      when "HTTPS_PROXY"; https_proxy = value
      when "NO_PROXY"; no_proxy = value
      when "HOST_PORT"; host_port = value.to_i
      when "HOST_IP_ADDR"; host_ip_addr = value
      when "GUEST_CPUS"; vm_num_cpus = value
      when "GUEST_MEMORY_MB"; vm_memory = value
      when "UBUNTU_MIRROR"; ubuntu_mirror = value
      when "VBOXADD_VERSION"; vboxadd_version = value
      end
    end
  end
  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: "127.0.0.1"
  if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
+    if http_proxy != ""
      config.proxy.http = http_proxy
    end
-    if !https_proxy.nil?
+    if https_proxy != ""
      config.proxy.https = https_proxy
    end
-    if !no_proxy.nil?
+    if https_proxy != ""
      config.proxy.no_proxy = no_proxy
    end
  elsif !http_proxy.nil? or !https_proxy.nil?
    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
    # We haven't figured out a workaround.
    puts "You have specified value for proxy in ~/.zulip-vagrant-config file but did not " \
         "install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install " \
         "vagrant-proxyconf` in a terminal.  This error will appear twice."
    exit
  end
-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
+  # Specify LXC provider before VirtualBox provider so it's preferred.
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
+  config.vm.provider "lxc" do |lxc|
-  # Specify Docker provider before VirtualBox provider so it's preferred.
+    if command? "lxc-ls"
-  config.vm.provider "docker" do |d, override|
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
-    override.vm.box = nil
+      if LXC_VERSION >= "1.1.0"
-    d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
-    d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
-    if !ubuntu_mirror.empty?
+        lxc.customize 'aa_allow_incomplete', 1
-      d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
+      end
      if LXC_VERSION >= "2.0.0"
        lxc.backingstore = 'dir'
      end
    end
    d.has_ssh = true
    d.create_args = ["--ulimit", "nofile=1024:65536"]
  end
  config.vm.provider "virtualbox" do |vb, override|
    override.vm.box = "ubuntu/trusty64"
    # It's possible we can get away with just 1.5GB; more testing needed
-    vb.memory = vm_memory
+    vb.memory = 2048
    vb.cpus = vm_num_cpus
    if !vboxadd_version.nil?
      override.vbguest.installer = Class.new(VagrantVbguest::Installers::Ubuntu) do
        define_method(:host_version) do |reload = false|
          VagrantVbguest::Version(vboxadd_version)
        end
      end
      override.vbguest.allow_downgrade = true
      override.vbguest.iso_path = "https://download.virtualbox.org/virtualbox/#{vboxadd_version}/VBoxGuestAdditions_#{vboxadd_version}.iso"
    end
  end
-  config.vm.provider "hyperv" do |h, override|
+$provision_script = <<SCRIPT
-    h.memory = vm_memory
+set -x
-    h.maxmemory = vm_memory
+set -e
-    h.cpus = vm_num_cpus
+set -o pipefail
-  end
+ln -nsf /srv/zulip ~/zulip
-
+/usr/bin/python /srv/zulip/tools/provision.py | sudo tee -a /var/log/zulip_provision.log
-  config.vm.provider "parallels" do |prl, override|
+SCRIPT
    prl.memory = vm_memory
    prl.cpus = vm_num_cpus
  end
  config.vm.provision "shell",
-    # We want provision to be run with the permissions of the vagrant user.
+    # We want provision.py to be run with the permissions of the vagrant user.
    privileged: false,
-    path: "tools/setup/vagrant-provision",
+    inline: $provision_script
    env: { "UBUNTU_MIRROR" => ubuntu_mirror }
 end
--- a/analytics/lib/counts.py
+++ b/analytics/lib/counts.py
@@ -1,821 +0,0 @@
 import logging
 import time
 from collections import OrderedDict, defaultdict
 from datetime import datetime, timedelta
 from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union
 from django.conf import settings
 from django.db import connection, models
 from django.db.models import F
 from psycopg2.sql import SQL, Composable, Identifier, Literal
 from analytics.models import (
    BaseCount,
    FillState,
    InstallationCount,
    RealmCount,
    StreamCount,
    UserCount,
    installation_epoch,
 )
 from zerver.lib.logging_util import log_to_file
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
 from zerver.models import Message, Realm, RealmAuditLog, Stream, UserActivityInterval, UserProfile
 ## Logging setup ##
 logger = logging.getLogger("zulip.management")
 log_to_file(logger, settings.ANALYTICS_LOG_PATH)
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365 * 1000)
 ## Class definitions ##
 class CountStat:
    HOUR = "hour"
    DAY = "day"
    FREQUENCIES = frozenset([HOUR, DAY])
    @property
    def time_increment(self) -> timedelta:
        if self.frequency == CountStat.HOUR:
            return timedelta(hours=1)
        return timedelta(days=1)
    def __init__(
        self,
        property: str,
        data_collector: "DataCollector",
        frequency: str,
        interval: Optional[timedelta] = None,
    ) -> None:
        self.property = property
        self.data_collector = data_collector
        # might have to do something different for bitfields
        if frequency not in self.FREQUENCIES:
            raise AssertionError(f"Unknown frequency: {frequency}")
        self.frequency = frequency
        if interval is not None:
            self.interval = interval
        else:
            self.interval = self.time_increment
    def __repr__(self) -> str:
        return f"<CountStat: {self.property}>"
    def last_successful_fill(self) -> Optional[datetime]:
        fillstate = FillState.objects.filter(property=self.property).first()
        if fillstate is None:
            return None
        if fillstate.state == FillState.DONE:
            return fillstate.end_time
        return fillstate.end_time - self.time_increment
 class LoggingCountStat(CountStat):
    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 class DependentCountStat(CountStat):
    def __init__(
        self,
        property: str,
        data_collector: "DataCollector",
        frequency: str,
        interval: Optional[timedelta] = None,
        dependencies: Sequence[str] = [],
    ) -> None:
        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
        self.dependencies = dependencies
 class DataCollector:
    def __init__(
        self,
        output_table: Type[BaseCount],
        pull_function: Optional[Callable[[str, datetime, datetime, Optional[Realm]], int]],
    ) -> None:
        self.output_table = output_table
        self.pull_function = pull_function
 ## CountStat-level operations ##
 def process_count_stat(
    stat: CountStat, fill_to_time: datetime, realm: Optional[Realm] = None
 ) -> None:
    # TODO: The realm argument is not yet supported, in that we don't
    # have a solution for how to update FillState if it is passed.  It
    # exists solely as partial plumbing for when we do fully implement
    # doing single-realm analytics runs for use cases like data import.
    #
    # Also, note that for the realm argument to be properly supported,
    # the CountStat object passed in needs to have come from
    # E.g. get_count_stats(realm), i.e. have the realm_id already
    # entered into the SQL query defined by the CountState object.
    verify_UTC(fill_to_time)
    if floor_to_hour(fill_to_time) != fill_to_time:
        raise ValueError(f"fill_to_time must be on an hour boundary: {fill_to_time}")
    fill_state = FillState.objects.filter(property=stat.property).first()
    if fill_state is None:
        currently_filled = installation_epoch()
        fill_state = FillState.objects.create(
            property=stat.property, end_time=currently_filled, state=FillState.DONE
        )
        logger.info("INITIALIZED %s %s", stat.property, currently_filled)
    elif fill_state.state == FillState.STARTED:
        logger.info("UNDO START %s %s", stat.property, fill_state.end_time)
        do_delete_counts_at_hour(stat, fill_state.end_time)
        currently_filled = fill_state.end_time - stat.time_increment
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        logger.info("UNDO DONE %s", stat.property)
    elif fill_state.state == FillState.DONE:
        currently_filled = fill_state.end_time
    else:
        raise AssertionError(f"Unknown value for FillState.state: {fill_state.state}.")
    if isinstance(stat, DependentCountStat):
        for dependency in stat.dependencies:
            dependency_fill_time = COUNT_STATS[dependency].last_successful_fill()
            if dependency_fill_time is None:
                logger.warning(
                    "DependentCountStat %s run before dependency %s.", stat.property, dependency
                )
                return
            fill_to_time = min(fill_to_time, dependency_fill_time)
    currently_filled = currently_filled + stat.time_increment
    while currently_filled <= fill_to_time:
        logger.info("START %s %s", stat.property, currently_filled)
        start = time.time()
        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
        do_fill_count_stat_at_hour(stat, currently_filled, realm)
        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
        end = time.time()
        currently_filled = currently_filled + stat.time_increment
        logger.info("DONE %s (%dms)", stat.property, (end - start) * 1000)
 def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
    fill_state.end_time = end_time
    fill_state.state = state
    fill_state.save()
 # We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
 # and is time-zone-aware. It is the caller's responsibility to enforce this!
 def do_fill_count_stat_at_hour(
    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
    start_time = end_time - stat.interval
    if not isinstance(stat, LoggingCountStat):
        timer = time.time()
        assert stat.data_collector.pull_function is not None
        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time, realm)
        logger.info(
            "%s run pull_function (%dms/%sr)",
            stat.property,
            (time.time() - timer) * 1000,
            rows_added,
        )
    do_aggregate_to_summary_table(stat, end_time, realm)
 def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
    if isinstance(stat, LoggingCountStat):
        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
        if stat.data_collector.output_table in [UserCount, StreamCount]:
            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
    else:
        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
 def do_aggregate_to_summary_table(
    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
    cursor = connection.cursor()
    # Aggregate into RealmCount
    output_table = stat.data_collector.output_table
    if realm is not None:
        realm_clause: Composable = SQL("AND zerver_realm.id = {}").format(Literal(realm.id))
    else:
        realm_clause = SQL("")
    if output_table in (UserCount, StreamCount):
        realmcount_query = SQL(
            """
            INSERT INTO analytics_realmcount
                (realm_id, value, property, subgroup, end_time)
            SELECT
                zerver_realm.id, COALESCE(sum({output_table}.value), 0), %(property)s,
                {output_table}.subgroup, %(end_time)s
            FROM zerver_realm
            JOIN {output_table}
            ON
                zerver_realm.id = {output_table}.realm_id
            WHERE
                {output_table}.property = %(property)s AND
                {output_table}.end_time = %(end_time)s
                {realm_clause}
            GROUP BY zerver_realm.id, {output_table}.subgroup
        """
        ).format(
            output_table=Identifier(output_table._meta.db_table),
            realm_clause=realm_clause,
        )
        start = time.time()
        cursor.execute(
            realmcount_query,
            {
                "property": stat.property,
                "end_time": end_time,
            },
        )
        end = time.time()
        logger.info(
            "%s RealmCount aggregation (%dms/%sr)",
            stat.property,
            (end - start) * 1000,
            cursor.rowcount,
        )
    if realm is None:
        # Aggregate into InstallationCount.  Only run if we just
        # processed counts for all realms.
        #
        # TODO: Add support for updating installation data after
        # changing an individual realm's values.
        installationcount_query = SQL(
            """
            INSERT INTO analytics_installationcount
                (value, property, subgroup, end_time)
            SELECT
                sum(value), %(property)s, analytics_realmcount.subgroup, %(end_time)s
            FROM analytics_realmcount
            WHERE
                property = %(property)s AND
                end_time = %(end_time)s
            GROUP BY analytics_realmcount.subgroup
        """
        )
        start = time.time()
        cursor.execute(
            installationcount_query,
            {
                "property": stat.property,
                "end_time": end_time,
            },
        )
        end = time.time()
        logger.info(
            "%s InstallationCount aggregation (%dms/%sr)",
            stat.property,
            (end - start) * 1000,
            cursor.rowcount,
        )
    cursor.close()
 ## Utility functions called from outside counts.py ##
 # called from zerver.actions; should not throw any errors
 def do_increment_logging_stat(
    zerver_object: Union[Realm, UserProfile, Stream],
    stat: CountStat,
    subgroup: Optional[Union[str, int, bool]],
    event_time: datetime,
    increment: int = 1,
 ) -> None:
    if not increment:
        return
    table = stat.data_collector.output_table
    if table == RealmCount:
        assert isinstance(zerver_object, Realm)
        id_args: Dict[str, Union[Realm, UserProfile, Stream]] = {"realm": zerver_object}
    elif table == UserCount:
        assert isinstance(zerver_object, UserProfile)
        id_args = {"realm": zerver_object.realm, "user": zerver_object}
    else:  # StreamCount
        assert isinstance(zerver_object, Stream)
        id_args = {"realm": zerver_object.realm, "stream": zerver_object}
    if stat.frequency == CountStat.DAY:
        end_time = ceiling_to_day(event_time)
    else:  # CountStat.HOUR:
        end_time = ceiling_to_hour(event_time)
    row, created = table.objects.get_or_create(
        property=stat.property,
        subgroup=subgroup,
        end_time=end_time,
        defaults={"value": increment},
        **id_args,
    )
    if not created:
        row.value = F("value") + increment
        row.save(update_fields=["value"])
 def do_drop_all_analytics_tables() -> None:
    UserCount.objects.all().delete()
    StreamCount.objects.all().delete()
    RealmCount.objects.all().delete()
    InstallationCount.objects.all().delete()
    FillState.objects.all().delete()
 def do_drop_single_stat(property: str) -> None:
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 ## DataCollector-level operations ##
 QueryFn = Callable[[Dict[str, Composable]], Composable]
 def do_pull_by_sql_query(
    property: str,
    start_time: datetime,
    end_time: datetime,
    query: QueryFn,
    group_by: Optional[Tuple[Type[models.Model], str]],
 ) -> int:
    if group_by is None:
        subgroup: Composable = SQL("NULL")
        group_by_clause: Composable = SQL("")
    else:
        subgroup = Identifier(group_by[0]._meta.db_table, group_by[1])
        group_by_clause = SQL(", {}").format(subgroup)
    # We do string replacement here because cursor.execute will reject a
    # group_by_clause given as a param.
    # We pass in the datetimes as params to cursor.execute so that we don't have to
    # think about how to convert python datetimes to SQL datetimes.
    query_ = query(
        {
            "subgroup": subgroup,
            "group_by_clause": group_by_clause,
        }
    )
    cursor = connection.cursor()
    cursor.execute(
        query_,
        {
            "property": property,
            "time_start": start_time,
            "time_end": end_time,
        },
    )
    rowcount = cursor.rowcount
    cursor.close()
    return rowcount
 def sql_data_collector(
    output_table: Type[BaseCount],
    query: QueryFn,
    group_by: Optional[Tuple[Type[models.Model], str]],
 ) -> DataCollector:
    def pull_function(
        property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
    ) -> int:
        # The pull function type needs to accept a Realm argument
        # because the 'minutes_active::day' CountStat uses
        # DataCollector directly for do_pull_minutes_active, which
        # requires the realm argument.  We ignore it here, because the
        # realm should have been already encoded in the `query` we're
        # passed.
        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
    return DataCollector(output_table, pull_function)
 def do_pull_minutes_active(
    property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
 ) -> int:
    user_activity_intervals = (
        UserActivityInterval.objects.filter(
            end__gt=start_time,
            start__lt=end_time,
        )
        .select_related(
            "user_profile",
        )
        .values_list("user_profile_id", "user_profile__realm_id", "start", "end")
    )
    seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
        if realm is None or realm.id == realm_id:
            start = max(start_time, interval_start)
            end = min(end_time, interval_end)
            seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
    rows = [
        UserCount(
            user_id=ids[0],
            realm_id=ids[1],
            property=property,
            end_time=end_time,
            value=int(seconds // 60),
        )
        for ids, seconds in seconds_active.items()
        if seconds >= 60
    ]
    UserCount.objects.bulk_create(rows)
    return len(rows)
 def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
        %(property)s, {subgroup}, %(time_end)s
    FROM zerver_userprofile
    JOIN zerver_message
    ON
        zerver_userprofile.id = zerver_message.sender_id
    WHERE
        zerver_userprofile.date_joined < %(time_end)s AND
        zerver_message.date_sent >= %(time_start)s AND
        {realm_clause}
        zerver_message.date_sent < %(time_end)s
    GROUP BY zerver_userprofile.id {group_by_clause}
 """
    ).format(**kwargs, realm_clause=realm_clause)
 # Note: ignores the group_by / group_by_clause.
 def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_usercount
            (realm_id, user_id, value, property, subgroup, end_time)
    SELECT realm_id, id, SUM(count) AS value, %(property)s, message_type, %(time_end)s
    FROM
    (
        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
        CASE WHEN
                  zerver_recipient.type = 1 THEN 'private_message'
             WHEN
                  zerver_recipient.type = 3 THEN 'huddle_message'
             WHEN
                  zerver_stream.invite_only = TRUE THEN 'private_stream'
             ELSE 'public_stream'
        END
        message_type
        FROM zerver_userprofile
        JOIN zerver_message
        ON
            zerver_userprofile.id = zerver_message.sender_id AND
            zerver_message.date_sent >= %(time_start)s AND
            {realm_clause}
            zerver_message.date_sent < %(time_end)s
        JOIN zerver_recipient
        ON
            zerver_message.recipient_id = zerver_recipient.id
        LEFT JOIN zerver_stream
        ON
            zerver_recipient.type_id = zerver_stream.id
        GROUP BY
            zerver_userprofile.realm_id, zerver_userprofile.id,
            zerver_recipient.type, zerver_stream.invite_only
    ) AS subquery
    GROUP BY realm_id, id, message_type
 """
    ).format(**kwargs, realm_clause=realm_clause)
 # This query joins to the UserProfile table since all current queries that
 # use this also subgroup on UserProfile.is_bot. If in the future there is a
 # stat that counts messages by stream and doesn't need the UserProfile
 # table, consider writing a new query for efficiency.
 def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_streamcount
        (stream_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_stream.id, zerver_stream.realm_id, count(*), %(property)s, {subgroup}, %(time_end)s
    FROM zerver_stream
    JOIN zerver_recipient
    ON
        zerver_stream.id = zerver_recipient.type_id
    JOIN zerver_message
    ON
        zerver_recipient.id = zerver_message.recipient_id
    JOIN zerver_userprofile
    ON
        zerver_message.sender_id = zerver_userprofile.id
    WHERE
        zerver_stream.date_created < %(time_end)s AND
        zerver_recipient.type = 2 AND
        zerver_message.date_sent >= %(time_start)s AND
        {realm_clause}
        zerver_message.date_sent < %(time_end)s
    GROUP BY zerver_stream.id {group_by_clause}
 """
    ).format(**kwargs, realm_clause=realm_clause)
 # Hardcodes the query needed by active_users:is_bot:day, since that is
 # currently the only stat that uses this.
 def count_user_by_realm_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
    FROM zerver_realm
    JOIN zerver_userprofile
    ON
        zerver_realm.id = zerver_userprofile.realm_id
    WHERE
        zerver_realm.date_created < %(time_end)s AND
        zerver_userprofile.date_joined >= %(time_start)s AND
        zerver_userprofile.date_joined < %(time_end)s AND
        {realm_clause}
        zerver_userprofile.is_active = TRUE
    GROUP BY zerver_realm.id {group_by_clause}
 """
    ).format(**kwargs, realm_clause=realm_clause)
 # Currently hardcodes the query needed for active_users_audit:is_bot:day.
 # Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
 # event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
 # In particular, it's important to ensure that migrations don't cause that to happen.
 def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        ral1.modified_user_id, ral1.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
    FROM zerver_realmauditlog ral1
    JOIN (
        SELECT modified_user_id, max(event_time) AS max_event_time
        FROM zerver_realmauditlog
        WHERE
            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
            {realm_clause}
            event_time < %(time_end)s
        GROUP BY modified_user_id
    ) ral2
    ON
        ral1.event_time = max_event_time AND
        ral1.modified_user_id = ral2.modified_user_id
    JOIN zerver_userprofile
    ON
        ral1.modified_user_id = zerver_userprofile.id
    WHERE
        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
    """
    ).format(
        **kwargs,
        user_created=Literal(RealmAuditLog.USER_CREATED),
        user_activated=Literal(RealmAuditLog.USER_ACTIVATED),
        user_deactivated=Literal(RealmAuditLog.USER_DEACTIVATED),
        user_reactivated=Literal(RealmAuditLog.USER_REACTIVATED),
        realm_clause=realm_clause,
    )
 def check_useractivityinterval_by_user_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_usercount
        (user_id, realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_userprofile.id, zerver_userprofile.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
    FROM zerver_userprofile
    JOIN zerver_useractivityinterval
    ON
        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
    WHERE
        zerver_useractivityinterval.end >= %(time_start)s AND
        {realm_clause}
        zerver_useractivityinterval.start < %(time_end)s
    GROUP BY zerver_userprofile.id {group_by_clause}
 """
    ).format(**kwargs, realm_clause=realm_clause)
 def count_realm_active_humans_query(realm: Optional[Realm]) -> QueryFn:
    if realm is None:
        realm_clause: Composable = SQL("")
    else:
        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
    return lambda kwargs: SQL(
        """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        usercount1.realm_id, count(*), %(property)s, NULL, %(time_end)s
    FROM (
        SELECT realm_id, user_id
        FROM analytics_usercount
        WHERE
            property = 'active_users_audit:is_bot:day' AND
            subgroup = 'false' AND
            {realm_clause}
            end_time = %(time_end)s
    ) usercount1
    JOIN (
        SELECT realm_id, user_id
        FROM analytics_usercount
        WHERE
            property = '15day_actives::day' AND
            {realm_clause}
            end_time = %(time_end)s
    ) usercount2
    ON
        usercount1.user_id = usercount2.user_id
    GROUP BY usercount1.realm_id
 """
    ).format(**kwargs, realm_clause=realm_clause)
 # Currently unused and untested
 count_stream_by_realm_query = lambda kwargs: SQL(
    """
    INSERT INTO analytics_realmcount
        (realm_id, value, property, subgroup, end_time)
    SELECT
        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
    FROM zerver_realm
    JOIN zerver_stream
    ON
        zerver_realm.id = zerver_stream.realm_id AND
    WHERE
        zerver_realm.date_created < %(time_end)s AND
        zerver_stream.date_created >= %(time_start)s AND
        zerver_stream.date_created < %(time_end)s
    GROUP BY zerver_realm.id {group_by_clause}
 """
 ).format(**kwargs)
 def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
    ## CountStat declarations ##
    count_stats_ = [
        # Messages sent stats
        # Stats that count the number of messages sent in various ways.
        # These are also the set of stats that read from the Message table.
        CountStat(
            "messages_sent:is_bot:hour",
            sql_data_collector(
                UserCount, count_message_by_user_query(realm), (UserProfile, "is_bot")
            ),
            CountStat.HOUR,
        ),
        CountStat(
            "messages_sent:message_type:day",
            sql_data_collector(UserCount, count_message_type_by_user_query(realm), None),
            CountStat.DAY,
        ),
        CountStat(
            "messages_sent:client:day",
            sql_data_collector(
                UserCount, count_message_by_user_query(realm), (Message, "sending_client_id")
            ),
            CountStat.DAY,
        ),
        CountStat(
            "messages_in_stream:is_bot:day",
            sql_data_collector(
                StreamCount, count_message_by_stream_query(realm), (UserProfile, "is_bot")
            ),
            CountStat.DAY,
        ),
        # Number of users stats
        # Stats that count the number of active users in the UserProfile.is_active sense.
        # 'active_users_audit:is_bot:day' is the canonical record of which users were
        # active on which days (in the UserProfile.is_active sense).
        # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
        CountStat(
            "active_users_audit:is_bot:day",
            sql_data_collector(
                UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
            ),
            CountStat.DAY,
        ),
        # Important note: LoggingCountStat objects aren't passed the
        # Realm argument, because by nature they have a logging
        # structure, not a pull-from-database structure, so there's no
        # way to compute them for a single realm after the fact (the
        # use case for passing a Realm argument).
        # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
        # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
        # sum sequence of 'active_users_log:is_bot:day', for any realm that
        # started after the latter stat was introduced.
        LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
        # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
        # approximation, e.g. if a user is deactivated between the end of the
        # day and when this stat is run, they won't be counted. However, is the
        # simplest of the three to inspect by hand.
        CountStat(
            "active_users:is_bot:day",
            sql_data_collector(
                RealmCount, count_user_by_realm_query(realm), (UserProfile, "is_bot")
            ),
            CountStat.DAY,
            interval=TIMEDELTA_MAX,
        ),
        # Messages read stats.  messages_read::hour is the total
        # number of messages read, whereas
        # messages_read_interactions::hour tries to count the total
        # number of UI interactions resulting in messages being marked
        # as read (imperfect because of batching of some request
        # types, but less likely to be overwhelmed by a single bulk
        # operation).
        LoggingCountStat("messages_read::hour", UserCount, CountStat.HOUR),
        LoggingCountStat("messages_read_interactions::hour", UserCount, CountStat.HOUR),
        # User activity stats
        # Stats that measure user activity in the UserActivityInterval sense.
        CountStat(
            "1day_actives::day",
            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
            CountStat.DAY,
            interval=timedelta(days=1) - UserActivityInterval.MIN_INTERVAL_LENGTH,
        ),
        CountStat(
            "7day_actives::day",
            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
            CountStat.DAY,
            interval=timedelta(days=7) - UserActivityInterval.MIN_INTERVAL_LENGTH,
        ),
        CountStat(
            "15day_actives::day",
            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
            CountStat.DAY,
            interval=timedelta(days=15) - UserActivityInterval.MIN_INTERVAL_LENGTH,
        ),
        CountStat(
            "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
        ),
        # Rate limiting stats
        # Used to limit the number of invitation emails sent by a realm
        LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
        # Dependent stats
        # Must come after their dependencies.
        # Canonical account of the number of active humans in a realm on each day.
        DependentCountStat(
            "realm_active_humans::day",
            sql_data_collector(RealmCount, count_realm_active_humans_query(realm), None),
            CountStat.DAY,
            dependencies=["active_users_audit:is_bot:day", "15day_actives::day"],
        ),
    ]
    return OrderedDict((stat.property, stat) for stat in count_stats_)
 # To avoid refactoring for now COUNT_STATS can be used as before
 COUNT_STATS = get_count_stats()
--- a/analytics/lib/fixtures.py
+++ b/analytics/lib/fixtures.py
@@ -1,79 +0,0 @@
 from math import sqrt
 from random import gauss, random, seed
 from typing import List
 from analytics.lib.counts import CountStat
 def generate_time_series_data(
    days: int = 100,
    business_hours_base: float = 10,
    non_business_hours_base: float = 10,
    growth: float = 1,
    autocorrelation: float = 0,
    spikiness: float = 1,
    holiday_rate: float = 0,
    frequency: str = CountStat.DAY,
    partial_sum: bool = False,
    random_seed: int = 26,
 ) -> List[int]:
    """
    Generate semi-realistic looking time series data for testing analytics graphs.
    days -- Number of days of data. Is the number of data points generated if
        frequency is CountStat.DAY.
    business_hours_base -- Average value during a business hour (or day) at beginning of
        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
    non_business_hours_base -- The above, for non-business hours/days.
    growth -- Ratio between average values at end of time series and beginning of time series.
    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
        point is unaffected by the previous point, and at 1 each point is a deterministic
        function of the previous point.
    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
        the variance.
    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
    frequency -- Should be CountStat.HOUR or CountStat.DAY.
    partial_sum -- If True, return partial sum of the series.
    random_seed -- Seed for random number generator.
    """
    if frequency == CountStat.HOUR:
        length = days * 24
        seasonality = [non_business_hours_base] * 24 * 7
        for day in range(5):
            for hour in range(8):
                seasonality[24 * day + hour] = business_hours_base
        holidays = []
        for i in range(days):
            holidays.extend([random() < holiday_rate] * 24)
    elif frequency == CountStat.DAY:
        length = days
        seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
            24 * non_business_hours_base
        ] * 2
        holidays = [random() < holiday_rate for i in range(days)]
    else:
        raise AssertionError(f"Unknown frequency: {frequency}")
    if length < 2:
        raise AssertionError(
            f"Must be generating at least 2 data points. Currently generating {length}"
        )
    growth_base = growth ** (1.0 / (length - 1))
    values_no_noise = [
        seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)
    ]
    seed(random_seed)
    noise_scalars = [gauss(0, 1)]
    for i in range(1, length):
        noise_scalars.append(
            noise_scalars[-1] * autocorrelation + gauss(0, 1) * (1 - autocorrelation)
        )
    values = [
        0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)
    ]
    if partial_sum:
        for i in range(1, length):
            values[i] = values[i - 1] + values[i]
    return [max(v, 0) for v in values]
--- a/analytics/lib/time_utils.py
+++ b/analytics/lib/time_utils.py
@@ -1,33 +0,0 @@
 from datetime import datetime, timedelta
 from typing import List, Optional
 from analytics.lib.counts import CountStat
 from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
 def time_range(
    start: datetime, end: datetime, frequency: str, min_length: Optional[int]
 ) -> List[datetime]:
    verify_UTC(start)
    verify_UTC(end)
    if frequency == CountStat.HOUR:
        end = floor_to_hour(end)
        step = timedelta(hours=1)
    elif frequency == CountStat.DAY:
        end = floor_to_day(end)
        step = timedelta(days=1)
    else:
        raise AssertionError(f"Unknown frequency: {frequency}")
    times = []
    if min_length is not None:
        start = min(start, end - (min_length - 1) * step)
    current = end
    while current >= start:
        times.append(current)
        current -= step
    return list(reversed(times))
--- a/analytics/management/commands/active_user_stats.py
+++ b/analytics/management/commands/active_user_stats.py
@@ -0,0 +1,60 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from django.core.management.base import BaseCommand
 from typing import Any
 from zerver.models import UserPresence, UserActivity
 from zerver.lib.utils import statsd, statsd_key
 from datetime import datetime, timedelta
 from collections import defaultdict
 class Command(BaseCommand):
    help = """Sends active user statistics to statsd.
    Run as a cron job that runs every 10 minutes."""
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        # Get list of all active users in the last 1 week
        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
        hour_buckets = [0.16, 2, 12, 24, 48, 168]
        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
        for last_presence in users:
            if last_presence.status == UserPresence.IDLE:
                known_active = last_presence.timestamp - timedelta(minutes=30)
            else:
                known_active = last_presence.timestamp
            for bucket in hour_buckets:
                if bucket not in user_info[last_presence.user_profile.realm.domain]:
                    user_info[last_presence.user_profile.realm.domain][bucket] = []
                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
                    user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
        for realm, buckets in user_info.items():
            print("Realm %s" % (realm,))
            for hr, users in sorted(buckets.items()):
                print("\tUsers for %s: %s" % (hr, len(users)))
                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
        # Also do stats for how many users have been reading the app.
        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
        user_info = defaultdict(dict)
        for activity in users_reading:
            for bucket in hour_buckets:
                if bucket not in user_info[activity.user_profile.realm.domain]:
                    user_info[activity.user_profile.realm.domain][bucket] = []
                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
                    user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
        for realm, buckets in user_info.items():
            print("Realm %s" % (realm,))
            for hr, users in sorted(buckets.items()):
                print("\tUsers reading for %s: %s" % (hr, len(users)))
                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
--- a/analytics/management/commands/active_user_stats_by_day.py
+++ b/analytics/management/commands/active_user_stats_by_day.py
@@ -0,0 +1,27 @@
 from __future__ import absolute_import
 from __future__ import print_function
 import datetime
 import pytz
 from optparse import make_option
 from typing import Any
 from django.core.management.base import BaseCommand
 from zerver.lib.statistics import activity_averages_during_day
 class Command(BaseCommand):
    help = "Generate statistics on user activity for a given day."
    option_list = BaseCommand.option_list + \
        (make_option('--date', default=None, action='store',
                     help="Day to query in format 2013-12-05.  Default is yesterday"),)
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options["date"] is None:
            date = datetime.datetime.now() - datetime.timedelta(days=1)
        else:
            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
        print("Activity data for", date)
        print(activity_averages_during_day(date))
        print("Please note that the total registered user count is a total for today")
--- a/analytics/management/commands/analyze_mit.py
+++ b/analytics/management/commands/analyze_mit.py
@@ -0,0 +1,86 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from typing import Any
 from optparse import make_option
 from django.core.management.base import BaseCommand
 from zerver.models import Recipient, Message
 from zerver.lib.timestamp import timestamp_to_datetime
 import datetime
 import time
 import logging
 def compute_stats(log_level):
    # type: (int) -> None
    logger = logging.getLogger()
    logger.setLevel(log_level)
    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
    mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
                                       recipient__type=Recipient.STREAM,
                                       pub_date__gt=one_week_ago)
    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
        mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
    # Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
    mit_query = mit_query.exclude(sender__email__contains=("/"))
    mit_query = mit_query.exclude(sender__email__contains=("aim.com"))
    mit_query = mit_query.exclude(
        sender__email__in=["rss@mit.edu", "bash@mit.edu", "apache@mit.edu",
                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
                           "root@mit.edu", "nagios@mit.edu",
                           "www-data|local-realm@mit.edu"])
    user_counts = {} # type: Dict[str, Dict[str, int]]
    for m in mit_query.select_related("sending_client", "sender"):
        email = m.sender.email
        user_counts.setdefault(email, {})
        user_counts[email].setdefault(m.sending_client.name, 0)
        user_counts[email][m.sending_client.name] += 1
    total_counts = {} # type: Dict[str, int]
    total_user_counts = {} # type: Dict[str, int]
    for email, counts in user_counts.items():
        total_user_counts.setdefault(email, 0)
        for client_name, count in counts.items():
            total_counts.setdefault(client_name, 0)
            total_counts[client_name] += count
            total_user_counts[email] += count
    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
    top_percents = {} # type: Dict[int, float]
    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
        top_percents[size] = 0.0
    for i, email in enumerate(sorted(total_user_counts.keys(),
                                     key=lambda x: -total_user_counts[x])):
        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
                               total_user_counts[email], 1)
        for size in top_percents.keys():
            top_percents.setdefault(size, 0)
            if i < size:
                top_percents[size] += (percent_zulip * 1.0 / size)
        logging.debug("%40s | %10s | %s%%" % (email, total_user_counts[email],
                                              percent_zulip))
    logging.info("")
    for size in sorted(top_percents.keys()):
        logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
    grand_total = sum(total_counts.values())
    print(grand_total)
    logging.info("%15s | %s" % ("Client", "Percentage"))
    for client in total_counts.keys():
        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
 class Command(BaseCommand):
    option_list = BaseCommand.option_list + \
        (make_option('--verbose', default=False, action='store_true'),)
    help = "Compute statistics on MIT Zephyr usage."
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        level = logging.INFO
        if options["verbose"]:
            level = logging.DEBUG
        compute_stats(level)
--- a/analytics/management/commands/analyze_user_activity.py
+++ b/analytics/management/commands/analyze_user_activity.py
@@ -0,0 +1,64 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from typing import Any, Dict
 from zerver.lib.statistics import seconds_usage_between
 from optparse import make_option
 from django.core.management.base import BaseCommand
 from zerver.models import UserProfile
 import datetime
 from django.utils.timezone import utc
 def analyze_activity(options):
    # type: (Dict[str, Any]) -> None
    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
    day_end = day_start + datetime.timedelta(days=options["duration"])
    user_profile_query = UserProfile.objects.all()
    if options["realm"]:
        user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
    print("Per-user online duration:\n")
    total_duration = datetime.timedelta(0)
    for user_profile in user_profile_query:
        duration = seconds_usage_between(user_profile, day_start, day_end)
        if duration == datetime.timedelta(0):
            continue
        total_duration += duration
        print("%-*s%s" % (37, user_profile.email, duration,))
    print("\nTotal Duration:                      %s" % (total_duration,))
    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
 class Command(BaseCommand):
    help = """Report analytics of user activity on a per-user and realm basis.
 This command aggregates user activity data that is collected by each user using Zulip. It attempts
 to approximate how much each user has been using Zulip per day, measured by recording each 15 minute
 period where some activity has occurred (mouse move or keyboard activity).
 It will correctly not count server-initiated reloads in the activity statistics.
 The duration flag can be used to control how many days to show usage duration for
 Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
 By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
 is shown for all realms"""
    option_list = BaseCommand.option_list + (
        make_option('--realm', action='store'),
        make_option('--date', action='store', default="2013-09-06"),
        make_option('--duration', action='store', default=1, type=int,
                    help="How many days to show usage information for"),
        )
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        analyze_activity(options)
--- a/analytics/management/commands/check_analytics_state.py
+++ b/analytics/management/commands/check_analytics_state.py
@@ -1,90 +0,0 @@
 import os
 import time
 from datetime import timedelta
 from typing import Any, Dict
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.models import installation_epoch
 from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
 from zerver.models import Realm
 states = {
    0: "OK",
    1: "WARNING",
    2: "CRITICAL",
    3: "UNKNOWN",
 }
 class Command(BaseCommand):
    help = """Checks FillState table.
    Run as a cron job that runs every hour."""
    def handle(self, *args: Any, **options: Any) -> None:
        fill_state = self.get_fill_state()
        status = fill_state["status"]
        message = fill_state["message"]
        state_file_path = "/var/lib/nagios_state/check-analytics-state"
        state_file_tmp = state_file_path + "-tmp"
        with open(state_file_tmp, "w") as f:
            f.write(f"{int(time.time())}|{status}|{states[status]}|{message}\n")
        os.rename(state_file_tmp, state_file_path)
    def get_fill_state(self) -> Dict[str, Any]:
        if not Realm.objects.exists():
            return {"status": 0, "message": "No realms exist, so not checking FillState."}
        warning_unfilled_properties = []
        critical_unfilled_properties = []
        for property, stat in COUNT_STATS.items():
            last_fill = stat.last_successful_fill()
            if last_fill is None:
                last_fill = installation_epoch()
            try:
                verify_UTC(last_fill)
            except TimeZoneNotUTCError:
                return {"status": 2, "message": f"FillState not in UTC for {property}"}
            if stat.frequency == CountStat.DAY:
                floor_function = floor_to_day
                warning_threshold = timedelta(hours=26)
                critical_threshold = timedelta(hours=50)
            else:  # CountStat.HOUR
                floor_function = floor_to_hour
                warning_threshold = timedelta(minutes=90)
                critical_threshold = timedelta(minutes=150)
            if floor_function(last_fill) != last_fill:
                return {
                    "status": 2,
                    "message": f"FillState not on {stat.frequency} boundary for {property}",
                }
            time_to_last_fill = timezone_now() - last_fill
            if time_to_last_fill > critical_threshold:
                critical_unfilled_properties.append(property)
            elif time_to_last_fill > warning_threshold:
                warning_unfilled_properties.append(property)
        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
            return {"status": 0, "message": "FillState looks fine."}
        if len(critical_unfilled_properties) == 0:
            return {
                "status": 1,
                "message": "Missed filling {} once.".format(
                    ", ".join(warning_unfilled_properties),
                ),
            }
        return {
            "status": 2,
            "message": "Missed filling {} once. Missed filling {} at least twice.".format(
                ", ".join(warning_unfilled_properties),
                ", ".join(critical_unfilled_properties),
            ),
        }
--- a/analytics/management/commands/clear_analytics_tables.py
+++ b/analytics/management/commands/clear_analytics_tables.py
@@ -1,21 +0,0 @@
 from argparse import ArgumentParser
 from typing import Any
 from django.core.management.base import BaseCommand, CommandError
 from analytics.lib.counts import do_drop_all_analytics_tables
 class Command(BaseCommand):
    help = """Clear analytics tables."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
    def handle(self, *args: Any, **options: Any) -> None:
        if options["force"]:
            do_drop_all_analytics_tables()
        else:
            raise CommandError(
                "Would delete all data from analytics tables (!); use --force to do so."
            )
--- a/analytics/management/commands/clear_single_stat.py
+++ b/analytics/management/commands/clear_single_stat.py
@@ -1,23 +0,0 @@
 from argparse import ArgumentParser
 from typing import Any
 from django.core.management.base import BaseCommand, CommandError
 from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 class Command(BaseCommand):
    help = """Clear analytics tables."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument("--force", action="store_true", help="Actually do it.")
        parser.add_argument("--property", help="The property of the stat to be cleared.")
    def handle(self, *args: Any, **options: Any) -> None:
        property = options["property"]
        if property not in COUNT_STATS:
            raise CommandError(f"Invalid property: {property}")
        if not options["force"]:
            raise CommandError("No action taken. Use --force.")
        do_drop_single_stat(property)
--- a/analytics/management/commands/client_activity.py
+++ b/analytics/management/commands/client_activity.py
@@ -0,0 +1,81 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from typing import Any
 from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
 from django.db.models import Count, QuerySet
 from zerver.models import UserActivity, UserProfile, Realm, \
    get_realm, get_user_profile_by_email
 import datetime
 class Command(BaseCommand):
    help = """Report rough client activity globally, for a realm, or for a user
 Usage examples:
 python manage.py client_activity
 python manage.py client_activity zulip.com
 python manage.py client_activity jesstess@zulip.com"""
    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
                            help="realm or user to estimate client activity for")
    def compute_activity(self, user_activity_objects):
        # type: (QuerySet) -> None
        # Report data from the past week.
        #
        # This is a rough report of client activity because we inconsistently
        # register activity from various clients; think of it as telling you
        # approximately how many people from a group have used a particular
        # client recently. For example, this might be useful to get a sense of
        # how popular different versions of a desktop client are.
        #
        # Importantly, this does NOT tell you anything about the relative
        # volumes of requests from clients.
        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
        client_counts = user_activity_objects.filter(
            last_visit__gt=threshold).values("client__name").annotate(
            count=Count('client__name'))
        total = 0
        counts = []
        for client_type in client_counts:
            count = client_type["count"]
            client = client_type["client__name"]
            total += count
            counts.append((count, client))
        counts.sort()
        for count in counts:
            print("%25s %15d" % (count[1], count[0]))
        print("Total:", total)
    def handle(self, *args, **options):
        # type: (*Any, **str) -> None
        if options['arg'] is None:
            # Report global activity.
            self.compute_activity(UserActivity.objects.all())
        else:
            arg = options['arg']
            try:
                # Report activity for a user.
                user_profile = get_user_profile_by_email(arg)
                self.compute_activity(UserActivity.objects.filter(
                        user_profile=user_profile))
            except UserProfile.DoesNotExist:
                try:
                    # Report activity for a realm.
                    realm = get_realm(arg)
                    self.compute_activity(UserActivity.objects.filter(
                            user_profile__realm=realm))
                except Realm.DoesNotExist:
                    print("Unknown user or domain %s" % (arg,))
                    exit(1)
--- a/analytics/management/commands/populate_analytics_db.py
+++ b/analytics/management/commands/populate_analytics_db.py
@@ -1,348 +0,0 @@
 import os
 from datetime import timedelta
 from typing import Any, Dict, List, Mapping, Type, Union
 from django.core.files.uploadedfile import UploadedFile
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
 from analytics.models import (
    BaseCount,
    FillState,
    InstallationCount,
    RealmCount,
    StreamCount,
    UserCount,
 )
 from zerver.actions.create_realm import do_create_realm
 from zerver.actions.users import do_change_user_role
 from zerver.lib.create_user import create_user
 from zerver.lib.storage import static_path
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
 from zerver.lib.upload import upload_message_attachment_from_request
 from zerver.models import (
    Client,
    Realm,
    RealmAuditLog,
    Recipient,
    Stream,
    Subscription,
    UserGroup,
    UserProfile,
 )
 class Command(BaseCommand):
    help = """Populates analytics tables with randomly generated data."""
    DAYS_OF_DATA = 100
    random_seed = 26
    def generate_fixture_data(
        self,
        stat: CountStat,
        business_hours_base: float,
        non_business_hours_base: float,
        growth: float,
        autocorrelation: float,
        spikiness: float,
        holiday_rate: float = 0,
        partial_sum: bool = False,
    ) -> List[int]:
        self.random_seed += 1
        return generate_time_series_data(
            days=self.DAYS_OF_DATA,
            business_hours_base=business_hours_base,
            non_business_hours_base=non_business_hours_base,
            growth=growth,
            autocorrelation=autocorrelation,
            spikiness=spikiness,
            holiday_rate=holiday_rate,
            frequency=stat.frequency,
            partial_sum=partial_sum,
            random_seed=self.random_seed,
        )
    def handle(self, *args: Any, **options: Any) -> None:
        # TODO: This should arguably only delete the objects
        # associated with the "analytics" realm.
        do_drop_all_analytics_tables()
        # This also deletes any objects with this realm as a foreign key
        Realm.objects.filter(string_id="analytics").delete()
        # Because we just deleted a bunch of objects in the database
        # directly (rather than deleting individual objects in Django,
        # in which case our post_save hooks would have flushed the
        # individual objects from memcached for us), we need to flush
        # memcached in order to ensure deleted objects aren't still
        # present in the memcached cache.
        from zerver.apps import flush_cache
        flush_cache(None)
        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
        last_end_time = floor_to_day(timezone_now())
        realm = do_create_realm(
            string_id="analytics", name="Analytics", date_created=installation_time
        )
        shylock = create_user(
            "shylock@analytics.ds",
            "Shylock",
            realm,
            full_name="Shylock",
            role=UserProfile.ROLE_REALM_OWNER,
            force_date_joined=installation_time,
        )
        do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None)
        # Create guest user for set_guest_users_statistic.
        create_user(
            "bassanio@analytics.ds",
            "Bassanio",
            realm,
            full_name="Bassanio",
            role=UserProfile.ROLE_GUEST,
            force_date_joined=installation_time,
        )
        administrators_user_group = UserGroup.objects.get(
            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=realm, is_system_group=True
        )
        stream = Stream.objects.create(
            name="all",
            realm=realm,
            date_created=installation_time,
            can_remove_subscribers_group=administrators_user_group,
        )
        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
        stream.recipient = recipient
        stream.save(update_fields=["recipient"])
        # Subscribe shylock to the stream to avoid invariant failures.
        Subscription.objects.create(
            recipient=recipient,
            user_profile=shylock,
            is_user_active=shylock.is_active,
            color=STREAM_ASSIGNMENT_COLORS[0],
        )
        RealmAuditLog.objects.create(
            realm=realm,
            modified_user=shylock,
            modified_stream=stream,
            event_last_message_id=0,
            event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
            event_time=installation_time,
        )
        # Create an attachment in the database for set_storage_space_used_statistic.
        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
        file_info = os.stat(IMAGE_FILE_PATH)
        file_size = file_info.st_size
        with open(IMAGE_FILE_PATH, "rb") as fp:
            upload_message_attachment_from_request(UploadedFile(fp), shylock, file_size)
        FixtureData = Mapping[Union[str, int, None], List[int]]
        def insert_fixture_data(
            stat: CountStat,
            fixture_data: FixtureData,
            table: Type[BaseCount],
        ) -> None:
            end_times = time_range(
                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
            )
            if table == InstallationCount:
                id_args: Dict[str, Any] = {}
            if table == RealmCount:
                id_args = {"realm": realm}
            if table == UserCount:
                id_args = {"realm": realm, "user": shylock}
            if table == StreamCount:
                id_args = {"stream": stream, "realm": realm}
            for subgroup, values in fixture_data.items():
                table.objects.bulk_create(
                    table(
                        property=stat.property,
                        subgroup=subgroup,
                        end_time=end_time,
                        value=value,
                        **id_args,
                    )
                    for end_time, value in zip(end_times, values)
                    if value != 0
                )
        stat = COUNT_STATS["1day_actives::day"]
        realm_data: FixtureData = {
            None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data: FixtureData = {
            None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["7day_actives::day"]
        realm_data = {
            None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["realm_active_humans::day"]
        realm_data = {
            None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["active_users_audit:is_bot:day"]
        realm_data = {
            "false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
            "true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
            "true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["messages_sent:is_bot:hour"]
        user_data: FixtureData = {
            "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["messages_sent:message_type:day"]
        user_data = {
            "public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
            "private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
            "huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            "public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
            "private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
            "private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
            "huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            "public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
            "private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
            "private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
            "huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        website, created = Client.objects.get_or_create(name="website")
        old_desktop, created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
        android, created = Client.objects.get_or_create(name="ZulipAndroid")
        iOS, created = Client.objects.get_or_create(name="ZulipiOS")
        react_native, created = Client.objects.get_or_create(name="ZulipMobile")
        API, created = Client.objects.get_or_create(name="API: Python")
        zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
        unused, created = Client.objects.get_or_create(name="unused")
        long_webhook, created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
        stat = COUNT_STATS["messages_sent:client:day"]
        user_data = {
            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
            zephyr_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {
            website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
            android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
            API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        installation_data = {
            website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
            android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
            API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
        }
        insert_fixture_data(stat, installation_data, InstallationCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
        realm_data = {
            "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
            "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
        }
        insert_fixture_data(stat, realm_data, RealmCount)
        stream_data: Mapping[Union[int, str, None], List[int]] = {
            "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
            "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
        }
        insert_fixture_data(stat, stream_data, StreamCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
        stat = COUNT_STATS["messages_read::hour"]
        user_data = {
            None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
        }
        insert_fixture_data(stat, user_data, UserCount)
        realm_data = {None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)}
        insert_fixture_data(stat, realm_data, RealmCount)
        FillState.objects.create(
            property=stat.property, end_time=last_end_time, state=FillState.DONE
        )
--- a/analytics/management/commands/realm_stats.py
+++ b/analytics/management/commands/realm_stats.py
@@ -0,0 +1,167 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from typing import Any
 from argparse import ArgumentParser
 import datetime
 import pytz
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
    Subscription, UserMessage, get_realm
 MOBILE_CLIENT_LIST = ["Android", "ios"]
 HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
 human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LIST)
 class Command(BaseCommand):
    help = "Generate statistics on realm activity."
    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")
    def active_users(self, realm):
        # type: (Realm) -> List[UserProfile]
        # Has been active (on the website, for now) in the last 7 days.
        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
        return [activity.user_profile for activity in \
                    UserActivity.objects.filter(user_profile__realm=realm,
                                                user_profile__is_active=True,
                                                last_visit__gt=activity_cutoff,
                                                query="/json/users/me/pointer",
                                                client__name="website")]
    def messages_sent_by(self, user, days_ago):
        # type: (UserProfile, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
    def total_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
    def human_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
    def api_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
    def stream_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
                                     recipient__type=Recipient.STREAM).count()
    def private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
    def group_private_messages(self, realm, days_ago):
        # type: (Realm, int) -> int
        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
    def report_percentage(self, numerator, denominator, text):
        # type: (float, float, str) -> None
        if not denominator:
            fraction = 0.0
        else:
            fraction = numerator / float(denominator)
        print("%.2f%% of" % (fraction * 100,), text)
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
            print(realm.domain)
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            active_users = self.active_users(realm)
            num_active = len(active_users)
            print("%d active users (%d total)" % (num_active, len(user_profiles)))
            streams = Stream.objects.filter(realm=realm).extra(
                tables=['zerver_subscription', 'zerver_recipient'],
                where=['zerver_subscription.recipient_id = zerver_recipient.id',
                       'zerver_recipient.type = 2',
                       'zerver_recipient.type_id = zerver_stream.id',
                       'zerver_subscription.active = true']).annotate(count=Count("name"))
            print("%d streams" % (streams.count(),))
            for days_ago in (1, 7, 30):
                print("In last %d days, users sent:" % (days_ago,))
                sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
                for quantity in sorted(sender_quantities, reverse=True):
                    print(quantity, end=' ')
                print("")
                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications == True])
            self.report_percentage(num_notifications_enabled, num_active,
                                   "active users have desktop notifications enabled")
            num_enter_sends = len([x for x in active_users if x.enter_sends])
            self.report_percentage(num_enter_sends, num_active,
                                   "active users have enter-sends")
            all_message_count = human_messages.filter(sender__realm=realm).count()
            multi_paragraph_message_count = human_messages.filter(
                sender__realm=realm, content__contains="\n\n").count()
            self.report_percentage(multi_paragraph_message_count, all_message_count,
                                   "all messages are multi-paragraph")
            # Starred messages
            starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
                                                  flags=UserMessage.flags.starred).values(
                "user_profile").annotate(count=Count("user_profile"))
            print("%d users have starred %d messages" % (
                len(starrers), sum([elt["count"] for elt in starrers])))
            active_user_subs = Subscription.objects.filter(
                user_profile__in=user_profiles, active=True)
            # Streams not in home view
            non_home_view = active_user_subs.filter(in_home_view=False).values(
                "user_profile").annotate(count=Count("user_profile"))
            print("%d users have %d streams not in home view" % (
                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
            # Code block markup
            markup_messages = human_messages.filter(
                sender__realm=realm, content__contains="~~~").values(
                "sender").annotate(count=Count("sender"))
            print("%d users have used code block markup on %s messages" % (
                len(markup_messages), sum([elt["count"] for elt in markup_messages])))
            # Notifications for stream messages
            notifications = active_user_subs.filter(notifications=True).values(
                "user_profile").annotate(count=Count("user_profile"))
            print("%d users receive desktop notifications for %d streams" % (
                len(notifications), sum([elt["count"] for elt in notifications])))
            print("")
--- a/analytics/management/commands/stream_stats.py
+++ b/analytics/management/commands/stream_stats.py
@@ -0,0 +1,46 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from typing import Any
 from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
 from django.db.models import Q
 from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
 class Command(BaseCommand):
    help = "Generate statistics on the streams for a realm."
    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")
    def handle(self, *args, **options):
        # type: (*Any, **str) -> None
        if options['realms']:
            try:
                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
            print(realm.domain)
            print("------------")
            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
            invite_only_count = 0
            for stream in streams:
                if stream.invite_only:
                    invite_only_count += 1
                    continue
                print("%25s" % (stream.name,), end=' ')
                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
                num_messages = len(Message.objects.filter(recipient=recipient))
                print("%12d" % (num_messages,))
            print("%d invite-only streams" % (invite_only_count,))
            print("")
--- a/analytics/management/commands/update_analytics_counts.py
+++ b/analytics/management/commands/update_analytics_counts.py
@@ -1,96 +0,0 @@
 import os
 import time
 from argparse import ArgumentParser
 from datetime import timezone
 from typing import Any, Dict
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
 from scripts.lib.zulip_tools import ENDC, WARNING
 from zerver.lib.remote_server import send_analytics_to_remote_server
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 class Command(BaseCommand):
    help = """Fills Analytics tables.
    Run as a cron job that runs every hour."""
    def add_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument(
            "--time",
            "-t",
            help="Update stat tables from current state to "
            "--time. Defaults to the current time.",
            default=timezone_now().isoformat(),
        )
        parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
        parser.add_argument(
            "--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
        )
        parser.add_argument(
            "--verbose", action="store_true", help="Print timing information to stdout."
        )
    def handle(self, *args: Any, **options: Any) -> None:
        try:
            os.mkdir(settings.ANALYTICS_LOCK_DIR)
        except OSError:
            print(
                f"{WARNING}Analytics lock {settings.ANALYTICS_LOCK_DIR} is unavailable;"
                f" exiting.{ENDC}"
            )
            return
        try:
            self.run_update_analytics_counts(options)
        finally:
            os.rmdir(settings.ANALYTICS_LOCK_DIR)
    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
        # installation_epoch relies on there being at least one realm; we
        # shouldn't run the analytics code if that condition isn't satisfied
        if not Realm.objects.exists():
            logger.info("No realms, stopping update_analytics_counts")
            return
        fill_to_time = parse_datetime(options["time"])
        assert fill_to_time is not None
        if options["utc"]:
            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
        if fill_to_time.tzinfo is None:
            raise ValueError(
                "--time must be time-zone-aware. Maybe you meant to use the --utc option?"
            )
        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
        if options["stat"] is not None:
            stats = [COUNT_STATS[options["stat"]]]
        else:
            stats = list(COUNT_STATS.values())
        logger.info("Starting updating analytics counts through %s", fill_to_time)
        if options["verbose"]:
            start = time.time()
            last = start
        for stat in stats:
            process_count_stat(stat, fill_to_time)
            if options["verbose"]:
                print(f"Updated {stat.property} in {time.time() - last:.3f}s")
                last = time.time()
        if options["verbose"]:
            print(
                f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
            )
        logger.info("Finished updating analytics counts through %s", fill_to_time)
        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
            send_analytics_to_remote_server()
--- a/analytics/management/commands/user_stats.py
+++ b/analytics/management/commands/user_stats.py
@@ -0,0 +1,48 @@
 from __future__ import absolute_import
 from __future__ import print_function
 from argparse import ArgumentParser
 import datetime
 import pytz
 from typing import Any
 from django.core.management.base import BaseCommand
 from zerver.models import UserProfile, Realm, Stream, Message, get_realm
 from six.moves import range
 class Command(BaseCommand):
    help = "Generate statistics on user activity."
    def add_arguments(self, parser):
        # type: (ArgumentParser) -> None
        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
                            help="realm to generate statistics for")
    def messages_sent_by(self, user, week):
        # type: (UserProfile, int) -> int
        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
    def handle(self, *args, **options):
        # type: (*Any, **Any) -> None
        if options['realms']:
            try:
                realms = [get_realm(domain) for domain in options['realms']]
            except Realm.DoesNotExist as e:
                print(e)
                exit(1)
        else:
            realms = Realm.objects.all()
        for realm in realms:
            print(realm.domain)
            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
            print("%d users" % (len(user_profiles),))
            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
            for user_profile in user_profiles:
                print("%35s" % (user_profile.email,), end=' ')
                for week in range(10):
                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
                print("")
--- a/analytics/migrations/0001_initial.py
+++ b/analytics/migrations/0001_initial.py
@@ -1,208 +0,0 @@
 import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("zerver", "0030_realm_org_type"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]
    operations = [
        migrations.CreateModel(
            name="Anomaly",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                ("info", models.CharField(max_length=1000)),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name="HuddleCount",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                (
                    "huddle",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
                    ),
                ),
                (
                    "user",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
                    ),
                ),
                ("property", models.CharField(max_length=40)),
                ("end_time", models.DateTimeField()),
                ("interval", models.CharField(max_length=20)),
                ("value", models.BigIntegerField()),
                (
                    "anomaly",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="analytics.Anomaly",
                        null=True,
                    ),
                ),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name="InstallationCount",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                ("property", models.CharField(max_length=40)),
                ("end_time", models.DateTimeField()),
                ("interval", models.CharField(max_length=20)),
                ("value", models.BigIntegerField()),
                (
                    "anomaly",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="analytics.Anomaly",
                        null=True,
                    ),
                ),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name="RealmCount",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                (
                    "realm",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
                    ),
                ),
                ("property", models.CharField(max_length=40)),
                ("end_time", models.DateTimeField()),
                ("interval", models.CharField(max_length=20)),
                ("value", models.BigIntegerField()),
                (
                    "anomaly",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="analytics.Anomaly",
                        null=True,
                    ),
                ),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name="StreamCount",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                (
                    "realm",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
                    ),
                ),
                (
                    "stream",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
                    ),
                ),
                ("property", models.CharField(max_length=40)),
                ("end_time", models.DateTimeField()),
                ("interval", models.CharField(max_length=20)),
                ("value", models.BigIntegerField()),
                (
                    "anomaly",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="analytics.Anomaly",
                        null=True,
                    ),
                ),
            ],
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name="UserCount",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                (
                    "realm",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
                    ),
                ),
                (
                    "user",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
                    ),
                ),
                ("property", models.CharField(max_length=40)),
                ("end_time", models.DateTimeField()),
                ("interval", models.CharField(max_length=20)),
                ("value", models.BigIntegerField()),
                (
                    "anomaly",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="analytics.Anomaly",
                        null=True,
                    ),
                ),
            ],
            bases=(models.Model,),
        ),
        migrations.AlterUniqueTogether(
            name="usercount",
            unique_together={("user", "property", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="streamcount",
            unique_together={("stream", "property", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="realmcount",
            unique_together={("realm", "property", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="installationcount",
            unique_together={("property", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="huddlecount",
            unique_together={("huddle", "property", "end_time", "interval")},
        ),
    ]
--- a/analytics/migrations/0002_remove_huddlecount.py
+++ b/analytics/migrations/0002_remove_huddlecount.py
@@ -1,29 +0,0 @@
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0001_initial"),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name="huddlecount",
            unique_together=set(),
        ),
        migrations.RemoveField(
            model_name="huddlecount",
            name="anomaly",
        ),
        migrations.RemoveField(
            model_name="huddlecount",
            name="huddle",
        ),
        migrations.RemoveField(
            model_name="huddlecount",
            name="user",
        ),
        migrations.DeleteModel(
            name="HuddleCount",
        ),
    ]
--- a/analytics/migrations/0003_fillstate.py
+++ b/analytics/migrations/0003_fillstate.py
@@ -1,26 +0,0 @@
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0002_remove_huddlecount"),
    ]
    operations = [
        migrations.CreateModel(
            name="FillState",
            fields=[
                (
                    "id",
                    models.AutoField(
                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
                    ),
                ),
                ("property", models.CharField(unique=True, max_length=40)),
                ("end_time", models.DateTimeField()),
                ("state", models.PositiveSmallIntegerField()),
                ("last_modified", models.DateTimeField(auto_now=True)),
            ],
            bases=(models.Model,),
        ),
    ]
--- a/analytics/migrations/0004_add_subgroup.py
+++ b/analytics/migrations/0004_add_subgroup.py
@@ -1,30 +0,0 @@
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0003_fillstate"),
    ]
    operations = [
        migrations.AddField(
            model_name="installationcount",
            name="subgroup",
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name="realmcount",
            name="subgroup",
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name="streamcount",
            name="subgroup",
            field=models.CharField(max_length=16, null=True),
        ),
        migrations.AddField(
            model_name="usercount",
            name="subgroup",
            field=models.CharField(max_length=16, null=True),
        ),
    ]
--- a/analytics/migrations/0005_alter_field_size.py
+++ b/analytics/migrations/0005_alter_field_size.py
@@ -1,50 +0,0 @@
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0004_add_subgroup"),
    ]
    operations = [
        migrations.AlterField(
            model_name="installationcount",
            name="interval",
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name="installationcount",
            name="property",
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name="realmcount",
            name="interval",
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name="realmcount",
            name="property",
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name="streamcount",
            name="interval",
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name="streamcount",
            name="property",
            field=models.CharField(max_length=32),
        ),
        migrations.AlterField(
            model_name="usercount",
            name="interval",
            field=models.CharField(max_length=8),
        ),
        migrations.AlterField(
            model_name="usercount",
            name="property",
            field=models.CharField(max_length=32),
        ),
    ]
--- a/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
+++ b/analytics/migrations/0006_add_subgroup_to_unique_constraints.py
@@ -1,26 +0,0 @@
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0005_alter_field_size"),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name="installationcount",
            unique_together={("property", "subgroup", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="realmcount",
            unique_together={("realm", "property", "subgroup", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="streamcount",
            unique_together={("stream", "property", "subgroup", "end_time", "interval")},
        ),
        migrations.AlterUniqueTogether(
            name="usercount",
            unique_together={("user", "property", "subgroup", "end_time", "interval")},
        ),
    ]
--- a/analytics/migrations/0007_remove_interval.py
+++ b/analytics/migrations/0007_remove_interval.py
@@ -1,43 +0,0 @@
 # Generated by Django 1.10.4 on 2017-01-16 20:50
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0006_add_subgroup_to_unique_constraints"),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name="installationcount",
            unique_together={("property", "subgroup", "end_time")},
        ),
        migrations.RemoveField(
            model_name="installationcount",
            name="interval",
        ),
        migrations.AlterUniqueTogether(
            name="realmcount",
            unique_together={("realm", "property", "subgroup", "end_time")},
        ),
        migrations.RemoveField(
            model_name="realmcount",
            name="interval",
        ),
        migrations.AlterUniqueTogether(
            name="streamcount",
            unique_together={("stream", "property", "subgroup", "end_time")},
        ),
        migrations.RemoveField(
            model_name="streamcount",
            name="interval",
        ),
        migrations.AlterUniqueTogether(
            name="usercount",
            unique_together={("user", "property", "subgroup", "end_time")},
        ),
        migrations.RemoveField(
            model_name="usercount",
            name="interval",
        ),
    ]
--- a/analytics/migrations/0008_add_count_indexes.py
+++ b/analytics/migrations/0008_add_count_indexes.py
@@ -1,24 +0,0 @@
 # Generated by Django 1.10.5 on 2017-02-01 22:28
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("zerver", "0050_userprofile_avatar_version"),
        ("analytics", "0007_remove_interval"),
    ]
    operations = [
        migrations.AlterIndexTogether(
            name="realmcount",
            index_together={("property", "end_time")},
        ),
        migrations.AlterIndexTogether(
            name="streamcount",
            index_together={("property", "realm", "end_time")},
        ),
        migrations.AlterIndexTogether(
            name="usercount",
            index_together={("property", "realm", "end_time")},
        ),
    ]
--- a/analytics/migrations/0009_remove_messages_to_stream_stat.py
+++ b/analytics/migrations/0009_remove_messages_to_stream_stat.py
@@ -1,30 +0,0 @@
 from django.db import migrations
 from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def delete_messages_sent_to_stream_stat(
    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
 ) -> None:
    UserCount = apps.get_model("analytics", "UserCount")
    StreamCount = apps.get_model("analytics", "StreamCount")
    RealmCount = apps.get_model("analytics", "RealmCount")
    InstallationCount = apps.get_model("analytics", "InstallationCount")
    FillState = apps.get_model("analytics", "FillState")
    property = "messages_sent_to_stream:is_bot"
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0008_add_count_indexes"),
    ]
    operations = [
        migrations.RunPython(delete_messages_sent_to_stream_stat),
    ]
--- a/analytics/migrations/0010_clear_messages_sent_values.py
+++ b/analytics/migrations/0010_clear_messages_sent_values.py
@@ -1,28 +0,0 @@
 from django.db import migrations
 from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def clear_message_sent_by_message_type_values(
    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
 ) -> None:
    UserCount = apps.get_model("analytics", "UserCount")
    StreamCount = apps.get_model("analytics", "StreamCount")
    RealmCount = apps.get_model("analytics", "RealmCount")
    InstallationCount = apps.get_model("analytics", "InstallationCount")
    FillState = apps.get_model("analytics", "FillState")
    property = "messages_sent:message_type:day"
    UserCount.objects.filter(property=property).delete()
    StreamCount.objects.filter(property=property).delete()
    RealmCount.objects.filter(property=property).delete()
    InstallationCount.objects.filter(property=property).delete()
    FillState.objects.filter(property=property).delete()
 class Migration(migrations.Migration):
    dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
    operations = [
        migrations.RunPython(clear_message_sent_by_message_type_values),
    ]
--- a/analytics/migrations/0011_clear_analytics_tables.py
+++ b/analytics/migrations/0011_clear_analytics_tables.py
@@ -1,27 +0,0 @@
 from django.db import migrations
 from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
    UserCount = apps.get_model("analytics", "UserCount")
    StreamCount = apps.get_model("analytics", "StreamCount")
    RealmCount = apps.get_model("analytics", "RealmCount")
    InstallationCount = apps.get_model("analytics", "InstallationCount")
    FillState = apps.get_model("analytics", "FillState")
    UserCount.objects.all().delete()
    StreamCount.objects.all().delete()
    RealmCount.objects.all().delete()
    InstallationCount.objects.all().delete()
    FillState.objects.all().delete()
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0010_clear_messages_sent_values"),
    ]
    operations = [
        migrations.RunPython(clear_analytics_tables),
    ]
--- a/analytics/migrations/0012_add_on_delete.py
+++ b/analytics/migrations/0012_add_on_delete.py
@@ -1,41 +0,0 @@
 # Generated by Django 1.11.6 on 2018-01-29 08:14
 import django.db.models.deletion
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0011_clear_analytics_tables"),
    ]
    operations = [
        migrations.AlterField(
            model_name="installationcount",
            name="anomaly",
            field=models.ForeignKey(
                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
            ),
        ),
        migrations.AlterField(
            model_name="realmcount",
            name="anomaly",
            field=models.ForeignKey(
                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
            ),
        ),
        migrations.AlterField(
            model_name="streamcount",
            name="anomaly",
            field=models.ForeignKey(
                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
            ),
        ),
        migrations.AlterField(
            model_name="usercount",
            name="anomaly",
            field=models.ForeignKey(
                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
            ),
        ),
    ]
--- a/analytics/migrations/0013_remove_anomaly.py
+++ b/analytics/migrations/0013_remove_anomaly.py
@@ -1,31 +0,0 @@
 # Generated by Django 1.11.18 on 2019-02-02 02:47
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0012_add_on_delete"),
    ]
    operations = [
        migrations.RemoveField(
            model_name="installationcount",
            name="anomaly",
        ),
        migrations.RemoveField(
            model_name="realmcount",
            name="anomaly",
        ),
        migrations.RemoveField(
            model_name="streamcount",
            name="anomaly",
        ),
        migrations.RemoveField(
            model_name="usercount",
            name="anomaly",
        ),
        migrations.DeleteModel(
            name="Anomaly",
        ),
    ]
--- a/analytics/migrations/0014_remove_fillstate_last_modified.py
+++ b/analytics/migrations/0014_remove_fillstate_last_modified.py
@@ -1,16 +0,0 @@
 # Generated by Django 1.11.26 on 2020-01-27 04:32
 from django.db import migrations
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0013_remove_anomaly"),
    ]
    operations = [
        migrations.RemoveField(
            model_name="fillstate",
            name="last_modified",
        ),
    ]
--- a/analytics/migrations/0015_clear_duplicate_counts.py
+++ b/analytics/migrations/0015_clear_duplicate_counts.py
@@ -1,64 +0,0 @@
 from django.db import migrations
 from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 from django.db.models import Count, Sum
 def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
    """This is a preparatory migration for our Analytics tables.
    The backstory is that Django's unique_together indexes do not properly
    handle the subgroup=None corner case (allowing duplicate rows that have a
    subgroup of None), which meant that in race conditions, rather than updating
    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
    create a duplicate row.
    In the next migration, we'll add a proper constraint to fix this bug, but
    we need to fix any existing problematic rows before we can add that constraint.
    We fix this in an appropriate fashion for each type of CountStat object; mainly
    this means deleting the extra rows, but for LoggingCountStat objects, we need to
    additionally combine the sums.
    """
    count_tables = dict(
        realm=apps.get_model("analytics", "RealmCount"),
        user=apps.get_model("analytics", "UserCount"),
        stream=apps.get_model("analytics", "StreamCount"),
        installation=apps.get_model("analytics", "InstallationCount"),
    )
    for name, count_table in count_tables.items():
        value = [name, "property", "end_time"]
        if name == "installation":
            value = ["property", "end_time"]
        counts = (
            count_table.objects.filter(subgroup=None)
            .values(*value)
            .annotate(Count("id"), Sum("value"))
            .filter(id__count__gt=1)
        )
        for count in counts:
            count.pop("id__count")
            total_value = count.pop("value__sum")
            duplicate_counts = list(count_table.objects.filter(**count))
            first_count = duplicate_counts[0]
            if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
                # For LoggingCountStat objects, the right fix is to combine the totals;
                # for other CountStat objects, we expect the duplicates to have the same value.
                # And so all we need to do is delete them.
                first_count.value = total_value
                first_count.save()
            to_cleanup = duplicate_counts[1:]
            for duplicate_count in to_cleanup:
                duplicate_count.delete()
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0014_remove_fillstate_last_modified"),
    ]
    operations = [
        migrations.RunPython(clear_duplicate_counts, reverse_code=migrations.RunPython.noop),
    ]
--- a/analytics/migrations/0016_unique_constraint_when_subgroup_null.py
+++ b/analytics/migrations/0016_unique_constraint_when_subgroup_null.py
@@ -1,92 +0,0 @@
 # Generated by Django 2.2.10 on 2020-02-29 19:40
 from django.db import migrations, models
 class Migration(migrations.Migration):
    dependencies = [
        ("analytics", "0015_clear_duplicate_counts"),
    ]
    operations = [
        migrations.AlterUniqueTogether(
            name="installationcount",
            unique_together=set(),
        ),
        migrations.AlterUniqueTogether(
            name="realmcount",
            unique_together=set(),
        ),
        migrations.AlterUniqueTogether(
            name="streamcount",
            unique_together=set(),
        ),
        migrations.AlterUniqueTogether(
            name="usercount",
            unique_together=set(),
        ),
        migrations.AddConstraint(
            model_name="installationcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=False),
                fields=("property", "subgroup", "end_time"),
                name="unique_installation_count",
            ),
        ),
        migrations.AddConstraint(
            model_name="installationcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=True),
                fields=("property", "end_time"),
                name="unique_installation_count_null_subgroup",
            ),
        ),
        migrations.AddConstraint(
            model_name="realmcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=False),
                fields=("realm", "property", "subgroup", "end_time"),
                name="unique_realm_count",
            ),
        ),
        migrations.AddConstraint(
            model_name="realmcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=True),
                fields=("realm", "property", "end_time"),
                name="unique_realm_count_null_subgroup",
            ),
        ),
        migrations.AddConstraint(
            model_name="streamcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=False),
                fields=("stream", "property", "subgroup", "end_time"),
                name="unique_stream_count",
            ),
        ),
        migrations.AddConstraint(
            model_name="streamcount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=True),
                fields=("stream", "property", "end_time"),
                name="unique_stream_count_null_subgroup",
            ),
        ),
        migrations.AddConstraint(
            model_name="usercount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=False),
                fields=("user", "property", "subgroup", "end_time"),
                name="unique_user_count",
            ),
        ),
        migrations.AddConstraint(
            model_name="usercount",
            constraint=models.UniqueConstraint(
                condition=models.Q(subgroup__isnull=True),
                fields=("user", "property", "end_time"),
                name="unique_user_count_null_subgroup",
            ),
        ),
    ]
--- a/analytics/models.py
+++ b/analytics/models.py
@@ -1,137 +0,0 @@
 import datetime
 from django.db import models
 from django.db.models import Q, UniqueConstraint
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Stream, UserProfile
 class FillState(models.Model):
    property = models.CharField(max_length=40, unique=True)
    end_time = models.DateTimeField()
    # Valid states are {DONE, STARTED}
    DONE = 1
    STARTED = 2
    state = models.PositiveSmallIntegerField()
    def __str__(self) -> str:
        return f"{self.property} {self.end_time} {self.state}"
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
 def installation_epoch() -> datetime.datetime:
    earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
        "date_created__min"
    ]
    return floor_to_day(earliest_realm_creation)
 class BaseCount(models.Model):
    # Note: When inheriting from BaseCount, you may want to rearrange
    # the order of the columns in the migration to make sure they
    # match how you'd like the table to be arranged.
    property = models.CharField(max_length=32)
    subgroup = models.CharField(max_length=16, null=True)
    end_time = models.DateTimeField()
    value = models.BigIntegerField()
    class Meta:
        abstract = True
 class InstallationCount(BaseCount):
    class Meta:
        # Handles invalid duplicate InstallationCount data
        constraints = [
            UniqueConstraint(
                fields=["property", "subgroup", "end_time"],
                condition=Q(subgroup__isnull=False),
                name="unique_installation_count",
            ),
            UniqueConstraint(
                fields=["property", "end_time"],
                condition=Q(subgroup__isnull=True),
                name="unique_installation_count_null_subgroup",
            ),
        ]
    def __str__(self) -> str:
        return f"{self.property} {self.subgroup} {self.value}"
 class RealmCount(BaseCount):
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        # Handles invalid duplicate RealmCount data
        constraints = [
            UniqueConstraint(
                fields=["realm", "property", "subgroup", "end_time"],
                condition=Q(subgroup__isnull=False),
                name="unique_realm_count",
            ),
            UniqueConstraint(
                fields=["realm", "property", "end_time"],
                condition=Q(subgroup__isnull=True),
                name="unique_realm_count_null_subgroup",
            ),
        ]
        index_together = ["property", "end_time"]
    def __str__(self) -> str:
        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
 class UserCount(BaseCount):
    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        # Handles invalid duplicate UserCount data
        constraints = [
            UniqueConstraint(
                fields=["user", "property", "subgroup", "end_time"],
                condition=Q(subgroup__isnull=False),
                name="unique_user_count",
            ),
            UniqueConstraint(
                fields=["user", "property", "end_time"],
                condition=Q(subgroup__isnull=True),
                name="unique_user_count_null_subgroup",
            ),
        ]
        # This index dramatically improves the performance of
        # aggregating from users to realms
        index_together = ["property", "realm", "end_time"]
    def __str__(self) -> str:
        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
 class StreamCount(BaseCount):
    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
    class Meta:
        # Handles invalid duplicate StreamCount data
        constraints = [
            UniqueConstraint(
                fields=["stream", "property", "subgroup", "end_time"],
                condition=Q(subgroup__isnull=False),
                name="unique_stream_count",
            ),
            UniqueConstraint(
                fields=["stream", "property", "end_time"],
                condition=Q(subgroup__isnull=True),
                name="unique_stream_count_null_subgroup",
            ),
        ]
        # This index dramatically improves the performance of
        # aggregating from streams to realms
        index_together = ["property", "realm", "end_time"]
    def __str__(self) -> str:
        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"
--- a/analytics/tests/test_activity_views.py
+++ b/analytics/tests/test_activity_views.py
@@ -1,48 +0,0 @@
 from unittest import mock
 from django.utils.timezone import now as timezone_now
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.models import Client, UserActivity, UserProfile, flush_per_request_caches
 class ActivityTest(ZulipTestCase):
    @mock.patch("stripe.Customer.list", return_value=[])
    def test_activity(self, unused_mock: mock.Mock) -> None:
        self.login("hamlet")
        client, _ = Client.objects.get_or_create(name="website")
        query = "/json/messages/flags"
        last_visit = timezone_now()
        count = 150
        for activity_user_profile in UserProfile.objects.all():
            UserActivity.objects.get_or_create(
                user_profile=activity_user_profile,
                client=client,
                query=query,
                count=count,
                last_visit=last_visit,
            )
        # Fails when not staff
        result = self.client_get("/activity")
        self.assertEqual(result.status_code, 302)
        user_profile = self.example_user("hamlet")
        user_profile.is_staff = True
        user_profile.save(update_fields=["is_staff"])
        flush_per_request_caches()
        with self.assert_database_query_count(18):
            result = self.client_get("/activity")
            self.assertEqual(result.status_code, 200)
        flush_per_request_caches()
        with self.assert_database_query_count(8):
            result = self.client_get("/realm_activity/zulip/")
            self.assertEqual(result.status_code, 200)
        iago = self.example_user("iago")
        flush_per_request_caches()
        with self.assert_database_query_count(5):
            result = self.client_get(f"/user_activity/{iago.id}/")
            self.assertEqual(result.status_code, 200)
--- a/analytics/tests/test_counts.py
+++ b/analytics/tests/test_counts.py
--- a/analytics/tests/test_fixtures.py
+++ b/analytics/tests/test_fixtures.py
@@ -1,40 +0,0 @@
 from analytics.lib.counts import CountStat
 from analytics.lib.fixtures import generate_time_series_data
 from zerver.lib.test_classes import ZulipTestCase
 # A very light test suite; the code being tested is not run in production.
 class TestFixtures(ZulipTestCase):
    def test_deterministic_settings(self) -> None:
        # test basic business_hour / non_business_hour calculation
        # test we get an array of the right length with frequency=CountStat.DAY
        data = generate_time_series_data(
            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0
        )
        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
        data = generate_time_series_data(
            days=1,
            business_hours_base=2000,
            non_business_hours_base=1500,
            growth=2,
            spikiness=0,
            frequency=CountStat.HOUR,
        )
        # test we get an array of the right length with frequency=CountStat.HOUR
        self.assert_length(data, 24)
        # test that growth doesn't affect the first data point
        self.assertEqual(data[0], 2000)
        # test that the last data point is growth times what it otherwise would be
        self.assertEqual(data[-1], 1500 * 2)
        # test autocorrelation == 1, since that's the easiest value to test
        data = generate_time_series_data(
            days=1,
            business_hours_base=2000,
            non_business_hours_base=2000,
            autocorrelation=1,
            frequency=CountStat.HOUR,
        )
        self.assertEqual(data[0], data[1])
        self.assertEqual(data[0], data[-1])
--- a/analytics/tests/test_stats_views.py
+++ b/analytics/tests/test_stats_views.py
@@ -1,629 +0,0 @@
 from datetime import datetime, timedelta, timezone
 from typing import List, Optional
 from django.utils.timezone import now as timezone_now
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
 from analytics.models import FillState, RealmCount, UserCount
 from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
 from zerver.models import Client, get_realm
 class TestStatsEndpoint(ZulipTestCase):
    def test_stats(self) -> None:
        self.user = self.example_user("hamlet")
        self.login_user(self.user)
        result = self.client_get("/stats")
        self.assertEqual(result.status_code, 200)
        # Check that we get something back
        self.assert_in_response("Zulip analytics for", result)
    def test_guest_user_cant_access_stats(self) -> None:
        self.user = self.example_user("polonius")
        self.login_user(self.user)
        result = self.client_get("/stats")
        self.assert_json_error(result, "Not allowed for guest users", 400)
        result = self.client_get("/json/analytics/chart_data")
        self.assert_json_error(result, "Not allowed for guest users", 400)
    def test_stats_for_realm(self) -> None:
        user = self.example_user("hamlet")
        self.login_user(user)
        result = self.client_get("/stats/realm/zulip/")
        self.assertEqual(result.status_code, 302)
        result = self.client_get("/stats/realm/not_existing_realm/")
        self.assertEqual(result.status_code, 302)
        user = self.example_user("hamlet")
        user.is_staff = True
        user.save(update_fields=["is_staff"])
        result = self.client_get("/stats/realm/not_existing_realm/")
        self.assertEqual(result.status_code, 404)
        result = self.client_get("/stats/realm/zulip/")
        self.assertEqual(result.status_code, 200)
        self.assert_in_response("Zulip analytics for", result)
    def test_stats_for_installation(self) -> None:
        user = self.example_user("hamlet")
        self.login_user(user)
        result = self.client_get("/stats/installation")
        self.assertEqual(result.status_code, 302)
        user = self.example_user("hamlet")
        user.is_staff = True
        user.save(update_fields=["is_staff"])
        result = self.client_get("/stats/installation")
        self.assertEqual(result.status_code, 200)
        self.assert_in_response("Zulip analytics for", result)
 class TestGetChartData(ZulipTestCase):
    def setUp(self) -> None:
        super().setUp()
        self.realm = get_realm("zulip")
        self.user = self.example_user("hamlet")
        self.login_user(self.user)
        self.end_times_hour = [
            ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
        ]
        self.end_times_day = [
            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
        ]
    def data(self, i: int) -> List[int]:
        return [0, 0, i, 0]
    def insert_data(
        self, stat: CountStat, realm_subgroups: List[Optional[str]], user_subgroups: List[str]
    ) -> None:
        if stat.frequency == CountStat.HOUR:
            insert_time = self.end_times_hour[2]
            fill_time = self.end_times_hour[-1]
        if stat.frequency == CountStat.DAY:
            insert_time = self.end_times_day[2]
            fill_time = self.end_times_day[-1]
        RealmCount.objects.bulk_create(
            RealmCount(
                property=stat.property,
                subgroup=subgroup,
                end_time=insert_time,
                value=100 + i,
                realm=self.realm,
            )
            for i, subgroup in enumerate(realm_subgroups)
        )
        UserCount.objects.bulk_create(
            UserCount(
                property=stat.property,
                subgroup=subgroup,
                end_time=insert_time,
                value=200 + i,
                realm=self.realm,
                user=self.user,
            )
            for i, subgroup in enumerate(user_subgroups)
        )
        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
    def test_number_of_humans(self) -> None:
        stat = COUNT_STATS["realm_active_humans::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["1day_actives::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["active_users_audit:is_bot:day"]
        self.insert_data(stat, ["false"], [])
        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
        data = self.assert_json_success(result)
        self.assertEqual(
            data,
            {
                "msg": "",
                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
                "frequency": CountStat.DAY,
                "everyone": {
                    "_1day": self.data(100),
                    "_15day": self.data(100),
                    "all_time": self.data(100),
                },
                "display_order": None,
                "result": "success",
            },
        )
    def test_messages_sent_over_time(self) -> None:
        stat = COUNT_STATS["messages_sent:is_bot:hour"]
        self.insert_data(stat, ["true", "false"], ["false"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data,
            {
                "msg": "",
                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
                "frequency": CountStat.HOUR,
                "everyone": {"bot": self.data(100), "human": self.data(101)},
                "user": {"bot": self.data(0), "human": self.data(200)},
                "display_order": None,
                "result": "success",
            },
        )
    def test_messages_sent_by_message_type(self) -> None:
        stat = COUNT_STATS["messages_sent:message_type:day"]
        self.insert_data(
            stat, ["public_stream", "private_message"], ["public_stream", "private_stream"]
        )
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data,
            {
                "msg": "",
                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
                "frequency": CountStat.DAY,
                "everyone": {
                    "Public streams": self.data(100),
                    "Private streams": self.data(0),
                    "Direct messages": self.data(101),
                    "Group direct messages": self.data(0),
                },
                "user": {
                    "Public streams": self.data(200),
                    "Private streams": self.data(201),
                    "Direct messages": self.data(0),
                    "Group direct messages": self.data(0),
                },
                "display_order": [
                    "Direct messages",
                    "Public streams",
                    "Private streams",
                    "Group direct messages",
                ],
                "result": "success",
            },
        )
    def test_messages_sent_by_client(self) -> None:
        stat = COUNT_STATS["messages_sent:client:day"]
        client1 = Client.objects.create(name="client 1")
        client2 = Client.objects.create(name="client 2")
        client3 = Client.objects.create(name="client 3")
        client4 = Client.objects.create(name="client 4")
        self.insert_data(
            stat,
            [str(client4.id), str(client3.id), str(client2.id)],
            [str(client3.id), str(client1.id)],
        )
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data,
            {
                "msg": "",
                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
                "frequency": CountStat.DAY,
                "everyone": {
                    "client 4": self.data(100),
                    "client 3": self.data(101),
                    "client 2": self.data(102),
                },
                "user": {"client 3": self.data(200), "client 1": self.data(201)},
                "display_order": ["client 1", "client 2", "client 3", "client 4"],
                "result": "success",
            },
        )
    def test_messages_read_over_time(self) -> None:
        stat = COUNT_STATS["messages_read::hour"]
        self.insert_data(stat, [None], [])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_read_over_time"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data,
            {
                "msg": "",
                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
                "frequency": CountStat.HOUR,
                "everyone": {"read": self.data(100)},
                "user": {"read": self.data(0)},
                "display_order": None,
                "result": "success",
            },
        )
    def test_include_empty_subgroups(self) -> None:
        FillState.objects.create(
            property="realm_active_humans::day",
            end_time=self.end_times_day[0],
            state=FillState.DONE,
        )
        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
        data = self.assert_json_success(result)
        self.assertEqual(data["everyone"], {"_1day": [0], "_15day": [0], "all_time": [0]})
        self.assertFalse("user" in data)
        FillState.objects.create(
            property="messages_sent:is_bot:hour",
            end_time=self.end_times_hour[0],
            state=FillState.DONE,
        )
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(data["everyone"], {"human": [0], "bot": [0]})
        self.assertEqual(data["user"], {"human": [0], "bot": [0]})
        FillState.objects.create(
            property="messages_sent:message_type:day",
            end_time=self.end_times_day[0],
            state=FillState.DONE,
        )
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data["everyone"],
            {
                "Public streams": [0],
                "Private streams": [0],
                "Direct messages": [0],
                "Group direct messages": [0],
            },
        )
        self.assertEqual(
            data["user"],
            {
                "Public streams": [0],
                "Private streams": [0],
                "Direct messages": [0],
                "Group direct messages": [0],
            },
        )
        FillState.objects.create(
            property="messages_sent:client:day",
            end_time=self.end_times_day[0],
            state=FillState.DONE,
        )
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
        )
        data = self.assert_json_success(result)
        self.assertEqual(data["everyone"], {})
        self.assertEqual(data["user"], {})
    def test_start_and_end(self) -> None:
        stat = COUNT_STATS["realm_active_humans::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["1day_actives::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["active_users_audit:is_bot:day"]
        self.insert_data(stat, ["false"], [])
        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
        # valid start and end
        result = self.client_get(
            "/json/analytics/chart_data",
            {
                "chart_name": "number_of_humans",
                "start": end_time_timestamps[1],
                "end": end_time_timestamps[2],
            },
        )
        data = self.assert_json_success(result)
        self.assertEqual(data["end_times"], end_time_timestamps[1:3])
        self.assertEqual(
            data["everyone"], {"_1day": [0, 100], "_15day": [0, 100], "all_time": [0, 100]}
        )
        # start later then end
        result = self.client_get(
            "/json/analytics/chart_data",
            {
                "chart_name": "number_of_humans",
                "start": end_time_timestamps[2],
                "end": end_time_timestamps[1],
            },
        )
        self.assert_json_error_contains(result, "Start time is later than")
    def test_min_length(self) -> None:
        stat = COUNT_STATS["realm_active_humans::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["1day_actives::day"]
        self.insert_data(stat, [None], [])
        stat = COUNT_STATS["active_users_audit:is_bot:day"]
        self.insert_data(stat, ["false"], [])
        # test min_length is too short to change anything
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 2}
        )
        data = self.assert_json_success(result)
        self.assertEqual(
            data["end_times"], [datetime_to_timestamp(dt) for dt in self.end_times_day]
        )
        self.assertEqual(
            data["everyone"],
            {"_1day": self.data(100), "_15day": self.data(100), "all_time": self.data(100)},
        )
        # test min_length larger than filled data
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 5}
        )
        data = self.assert_json_success(result)
        end_times = [
            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)
        ]
        self.assertEqual(data["end_times"], [datetime_to_timestamp(dt) for dt in end_times])
        self.assertEqual(
            data["everyone"],
            {
                "_1day": [0, *self.data(100)],
                "_15day": [0, *self.data(100)],
                "all_time": [0, *self.data(100)],
            },
        )
    def test_non_existent_chart(self) -> None:
        result = self.client_get("/json/analytics/chart_data", {"chart_name": "does_not_exist"})
        self.assert_json_error_contains(result, "Unknown chart name")
    def test_analytics_not_running(self) -> None:
        realm = get_realm("zulip")
        self.assertEqual(FillState.objects.count(), 0)
        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        with self.assertLogs(level="WARNING") as m:
            result = self.client_get(
                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
            )
            self.assertEqual(
                m.output,
                [
                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
                ],
            )
        self.assert_json_error_contains(result, "No analytics data available")
        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
        realm.save(update_fields=["date_created"])
        with self.assertLogs(level="WARNING") as m:
            result = self.client_get(
                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
            )
            self.assertEqual(
                m.output,
                [
                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
                ],
            )
        self.assert_json_error_contains(result, "No analytics data available")
        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        self.assert_json_success(result)
        realm.date_created = timezone_now() - timedelta(hours=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        self.assert_json_success(result)
        end_time = timezone_now() - timedelta(days=5)
        fill_state = FillState.objects.create(
            property="messages_sent:is_bot:hour", end_time=end_time, state=FillState.DONE
        )
        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        with self.assertLogs(level="WARNING") as m:
            result = self.client_get(
                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
            )
            self.assertEqual(
                m.output,
                [
                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
                ],
            )
        self.assert_json_error_contains(result, "No analytics data available")
        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        self.assert_json_success(result)
        end_time = timezone_now() - timedelta(days=2)
        fill_state.end_time = end_time
        fill_state.save(update_fields=["end_time"])
        realm.date_created = timezone_now() - timedelta(days=3)
        realm.save(update_fields=["date_created"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        self.assert_json_success(result)
        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
        realm.save(update_fields=["date_created"])
        with self.assertLogs(level="WARNING") as m:
            result = self.client_get(
                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
            )
            self.assertEqual(
                m.output,
                [
                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
                ],
            )
        self.assert_json_error_contains(result, "No analytics data available")
        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
        realm.save(update_fields=["date_created"])
        result = self.client_get(
            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
        )
        self.assert_json_success(result)
    def test_get_chart_data_for_realm(self) -> None:
        user = self.example_user("hamlet")
        self.login_user(user)
        result = self.client_get(
            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
        )
        self.assert_json_error(result, "Must be an server administrator", 400)
        user = self.example_user("hamlet")
        user.is_staff = True
        user.save(update_fields=["is_staff"])
        stat = COUNT_STATS["realm_active_humans::day"]
        self.insert_data(stat, [None], [])
        result = self.client_get(
            "/json/analytics/chart_data/realm/not_existing_realm",
            {"chart_name": "number_of_humans"},
        )
        self.assert_json_error(result, "Invalid organization", 400)
        result = self.client_get(
            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
        )
        self.assert_json_success(result)
    def test_get_chart_data_for_installation(self) -> None:
        user = self.example_user("hamlet")
        self.login_user(user)
        result = self.client_get(
            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
        )
        self.assert_json_error(result, "Must be an server administrator", 400)
        user = self.example_user("hamlet")
        user.is_staff = True
        user.save(update_fields=["is_staff"])
        stat = COUNT_STATS["realm_active_humans::day"]
        self.insert_data(stat, [None], [])
        result = self.client_get(
            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
        )
        self.assert_json_success(result)
 class TestGetChartDataHelpers(ZulipTestCase):
    def test_sort_by_totals(self) -> None:
        empty: List[int] = []
        value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
        self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
    def test_sort_client_labels(self) -> None:
        data = {
            "everyone": {"a": [16], "c": [15], "b": [14], "e": [13], "d": [12], "h": [11]},
            "user": {"a": [6], "b": [5], "d": [4], "e": [3], "f": [2], "g": [1]},
        }
        self.assertEqual(sort_client_labels(data), ["a", "b", "c", "d", "e", "f", "g", "h"])
 class TestTimeRange(ZulipTestCase):
    def test_time_range(self) -> None:
        HOUR = timedelta(hours=1)
        DAY = timedelta(days=1)
        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
        # test start == end
        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
        # test start == end == boundary, and min_length == 0
        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
        # test start and end on different boundaries
        self.assertEqual(
            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, None),
            [floor_hour, floor_hour + HOUR],
        )
        self.assertEqual(
            time_range(floor_day, floor_day + DAY, CountStat.DAY, None),
            [floor_day, floor_day + DAY],
        )
        # test min_length
        self.assertEqual(
            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, 4),
            [floor_hour - 2 * HOUR, floor_hour - HOUR, floor_hour, floor_hour + HOUR],
        )
        self.assertEqual(
            time_range(floor_day, floor_day + DAY, CountStat.DAY, 4),
            [floor_day - 2 * DAY, floor_day - DAY, floor_day, floor_day + DAY],
        )
 class TestMapArrays(ZulipTestCase):
    def test_map_arrays(self) -> None:
        a = {
            "desktop app 1.0": [1, 2, 3],
            "desktop app 2.0": [10, 12, 13],
            "desktop app 3.0": [21, 22, 23],
            "website": [1, 2, 3],
            "ZulipiOS": [1, 2, 3],
            "ZulipElectron": [2, 5, 7],
            "ZulipMobile": [1, 5, 7],
            "ZulipPython": [1, 2, 3],
            "API: Python": [1, 2, 3],
            "SomethingRandom": [4, 5, 6],
            "ZulipGitHubWebhook": [7, 7, 9],
            "ZulipAndroid": [64, 63, 65],
            "ZulipTerminal": [9, 10, 11],
        }
        result = rewrite_client_arrays(a)
        self.assertEqual(
            result,
            {
                "Old desktop app": [32, 36, 39],
                "Old iOS app": [1, 2, 3],
                "Desktop app": [2, 5, 7],
                "Mobile app": [1, 5, 7],
                "Web app": [1, 2, 3],
                "Python API": [2, 4, 6],
                "SomethingRandom": [4, 5, 6],
                "GitHub webhook": [7, 7, 9],
                "Old Android app": [64, 63, 65],
                "Terminal app": [9, 10, 11],
            },
        )
--- a/analytics/tests/test_support_views.py
+++ b/analytics/tests/test_support_views.py
@@ -1,741 +0,0 @@
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Optional
 from unittest import mock
 import orjson
 from django.utils.timezone import now as timezone_now
 from corporate.lib.stripe import add_months, update_sponsorship_status
 from corporate.models import Customer, CustomerPlan, LicenseLedger, get_customer_by_realm
 from zerver.actions.invites import do_create_multiuse_invite_link
 from zerver.actions.realm_settings import do_change_realm_org_type, do_send_realm_reactivation_email
 from zerver.actions.user_settings import do_change_user_setting
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm
 from zerver.models import (
    MultiuseInvite,
    PreregistrationUser,
    Realm,
    UserMessage,
    UserProfile,
    get_org_type_display_name,
    get_realm,
 )
 if TYPE_CHECKING:
    from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
 class TestSupportEndpoint(ZulipTestCase):
    def test_search(self) -> None:
        reset_email_visibility_to_everyone_in_zulip_realm()
        lear_user = self.lear_user("king")
        lear_user.is_staff = True
        lear_user.save(update_fields=["is_staff"])
        lear_realm = get_realm("lear")
        def assert_user_details_in_html_response(
            html_response: "TestHttpResponse", full_name: str, email: str, role: str
        ) -> None:
            self.assert_in_success_response(
                [
                    '<span class="label">user</span>\n',
                    f"<h3>{full_name}</h3>",
                    f"<b>Email</b>: {email}",
                    "<b>Is active</b>: True<br />",
                    f"<b>Role</b>: {role}<br />",
                ],
                html_response,
            )
        def create_invitation(
            stream: str, invitee_email: str, realm: Optional[Realm] = None
        ) -> None:
            invite_expires_in_minutes = 10 * 24 * 60
            self.client_post(
                "/json/invites",
                {
                    "invitee_emails": [invitee_email],
                    "stream_ids": orjson.dumps([self.get_stream_id(stream, realm)]).decode(),
                    "invite_expires_in_minutes": invite_expires_in_minutes,
                    "invite_as": PreregistrationUser.INVITE_AS["MEMBER"],
                },
                subdomain=realm.string_id if realm is not None else "zulip",
            )
        def check_hamlet_user_query_result(result: "TestHttpResponse") -> None:
            assert_user_details_in_html_response(
                result, "King Hamlet", self.example_email("hamlet"), "Member"
            )
            self.assert_in_success_response(
                [
                    f"<b>Admins</b>: {self.example_email('iago')}\n",
                    f"<b>Owners</b>: {self.example_email('desdemona')}\n",
                    'class="copy-button" data-copytext="{}">'.format(self.example_email("iago")),
                    'class="copy-button" data-copytext="{}">'.format(
                        self.example_email("desdemona")
                    ),
                ],
                result,
            )
        def check_lear_user_query_result(result: "TestHttpResponse") -> None:
            assert_user_details_in_html_response(
                result, lear_user.full_name, lear_user.email, "Member"
            )
        def check_othello_user_query_result(result: "TestHttpResponse") -> None:
            assert_user_details_in_html_response(
                result, "Othello, the Moor of Venice", self.example_email("othello"), "Member"
            )
        def check_polonius_user_query_result(result: "TestHttpResponse") -> None:
            assert_user_details_in_html_response(
                result, "Polonius", self.example_email("polonius"), "Guest"
            )
        def check_zulip_realm_query_result(result: "TestHttpResponse") -> None:
            zulip_realm = get_realm("zulip")
            first_human_user = zulip_realm.get_first_human_user()
            assert first_human_user is not None
            self.assert_in_success_response(
                [
                    f"<b>First human user</b>: {first_human_user.delivery_email}\n",
                    f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"',
                    "Zulip Dev</h3>",
                    '<option value="1" selected>Self-hosted</option>',
                    '<option value="2" >Limited</option>',
                    'input type="number" name="discount" value="None"',
                    '<option value="active" selected>Active</option>',
                    '<option value="deactivated" >Deactivated</option>',
                    f'<option value="{zulip_realm.org_type}" selected>',
                    'scrub-realm-button">',
                    'data-string-id="zulip"',
                ],
                result,
            )
        def check_lear_realm_query_result(result: "TestHttpResponse") -> None:
            self.assert_in_success_response(
                [
                    f'<input type="hidden" name="realm_id" value="{lear_realm.id}"',
                    "Lear &amp; Co.</h3>",
                    '<option value="1" selected>Self-hosted</option>',
                    '<option value="2" >Limited</option>',
                    'input type="number" name="discount" value="None"',
                    '<option value="active" selected>Active</option>',
                    '<option value="deactivated" >Deactivated</option>',
                    'scrub-realm-button">',
                    'data-string-id="lear"',
                    "<b>Name</b>: Zulip Cloud Standard",
                    "<b>Status</b>: Active",
                    "<b>Billing schedule</b>: Annual",
                    "<b>Licenses</b>: 2/10 (Manual)",
                    "<b>Price per license</b>: $80.0",
                    "<b>Next invoice date</b>: 02 January 2017",
                    '<option value="send_invoice" selected>',
                    '<option value="charge_automatically" >',
                ],
                result,
            )
        def check_preregistration_user_query_result(
            result: "TestHttpResponse", email: str, invite: bool = False
        ) -> None:
            self.assert_in_success_response(
                [
                    '<span class="label">preregistration user</span>\n',
                    f"<b>Email</b>: {email}",
                ],
                result,
            )
            if invite:
                self.assert_in_success_response(['<span class="label">invite</span>'], result)
                self.assert_in_success_response(
                    [
                        "<b>Expires in</b>: 1\xa0week, 3\xa0days",
                        "<b>Status</b>: Link has not been used",
                    ],
                    result,
                )
                self.assert_in_success_response([], result)
            else:
                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
                self.assert_in_success_response(
                    [
                        "<b>Expires in</b>: 1\xa0day",
                        "<b>Status</b>: Link has not been used",
                    ],
                    result,
                )
        def check_realm_creation_query_result(result: "TestHttpResponse", email: str) -> None:
            self.assert_in_success_response(
                [
                    '<span class="label">preregistration user</span>\n',
                    '<span class="label">realm creation</span>\n',
                    "<b>Link</b>: http://testserver/accounts/do_confirm/",
                    "<b>Expires in</b>: 1\xa0day",
                ],
                result,
            )
        def check_multiuse_invite_link_query_result(result: "TestHttpResponse") -> None:
            self.assert_in_success_response(
                [
                    '<span class="label">multiuse invite</span>\n',
                    "<b>Link</b>: http://zulip.testserver/join/",
                    "<b>Expires in</b>: 1\xa0week, 3\xa0days",
                ],
                result,
            )
        def check_realm_reactivation_link_query_result(result: "TestHttpResponse") -> None:
            self.assert_in_success_response(
                [
                    '<span class="label">realm reactivation</span>\n',
                    "<b>Link</b>: http://zulip.testserver/reactivate/",
                    "<b>Expires in</b>: 1\xa0day",
                ],
                result,
            )
        def get_check_query_result(
            query: str, count: int, subdomain: str = "zulip"
        ) -> "TestHttpResponse":
            result = self.client_get("/activity/support", {"q": query}, subdomain=subdomain)
            self.assertEqual(result.content.decode().count("support-query-result"), count)
            return result
        self.login("cordelia")
        result = self.client_get("/activity/support")
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        self.login("iago")
        do_change_user_setting(
            self.example_user("hamlet"),
            "email_address_visibility",
            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
            acting_user=None,
        )
        customer = Customer.objects.create(realm=lear_realm, stripe_customer_id="cus_123")
        now = datetime(2016, 1, 2, tzinfo=timezone.utc)
        plan = CustomerPlan.objects.create(
            customer=customer,
            billing_cycle_anchor=now,
            billing_schedule=CustomerPlan.ANNUAL,
            tier=CustomerPlan.STANDARD,
            price_per_license=8000,
            next_invoice_date=add_months(now, 12),
        )
        LicenseLedger.objects.create(
            licenses=10,
            licenses_at_next_renewal=10,
            event_time=timezone_now(),
            is_renewal=True,
            plan=plan,
        )
        result = self.client_get("/activity/support")
        self.assert_in_success_response(
            ['<input type="text" name="q" class="input-xxlarge search-query"'], result
        )
        result = get_check_query_result(self.example_email("hamlet"), 1)
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        # Search should be case-insensitive:
        assert self.example_email("hamlet") != self.example_email("hamlet").upper()
        result = get_check_query_result(self.example_email("hamlet").upper(), 1)
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        result = get_check_query_result(lear_user.email, 1)
        check_lear_user_query_result(result)
        check_lear_realm_query_result(result)
        result = get_check_query_result(self.example_email("polonius"), 1)
        check_polonius_user_query_result(result)
        check_zulip_realm_query_result(result)
        result = get_check_query_result("lear", 1)
        check_lear_realm_query_result(result)
        result = get_check_query_result("http://lear.testserver", 1)
        check_lear_realm_query_result(result)
        with self.settings(REALM_HOSTS={"zulip": "localhost"}):
            result = get_check_query_result("http://localhost", 1)
            check_zulip_realm_query_result(result)
        result = get_check_query_result("hamlet@zulip.com, lear", 2)
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        check_lear_realm_query_result(result)
        result = get_check_query_result("King hamlet,lear", 2)
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        check_lear_realm_query_result(result)
        result = get_check_query_result("Othello, the Moor of Venice", 1)
        check_othello_user_query_result(result)
        check_zulip_realm_query_result(result)
        result = get_check_query_result("lear, Hamlet <hamlet@zulip.com>", 2)
        check_hamlet_user_query_result(result)
        check_zulip_realm_query_result(result)
        check_lear_realm_query_result(result)
        with mock.patch(
            "analytics.views.support.timezone_now",
            return_value=timezone_now() - timedelta(minutes=50),
        ):
            self.client_post("/accounts/home/", {"email": self.nonreg_email("test")})
            self.login("iago")
            result = get_check_query_result(self.nonreg_email("test"), 1)
            check_preregistration_user_query_result(result, self.nonreg_email("test"))
            check_zulip_realm_query_result(result)
            create_invitation("Denmark", self.nonreg_email("test1"))
            result = get_check_query_result(self.nonreg_email("test1"), 1)
            check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
            check_zulip_realm_query_result(result)
            email = self.nonreg_email("alice")
            self.submit_realm_creation_form(
                email, realm_subdomain="zuliptest", realm_name="Zulip test"
            )
            result = get_check_query_result(email, 1)
            check_realm_creation_query_result(result, email)
            invite_expires_in_minutes = 10 * 24 * 60
            do_create_multiuse_invite_link(
                self.example_user("hamlet"),
                invited_as=1,
                invite_expires_in_minutes=invite_expires_in_minutes,
            )
            result = get_check_query_result("zulip", 2)
            check_multiuse_invite_link_query_result(result)
            check_zulip_realm_query_result(result)
            MultiuseInvite.objects.all().delete()
            do_send_realm_reactivation_email(get_realm("zulip"), acting_user=None)
            result = get_check_query_result("zulip", 2)
            check_realm_reactivation_link_query_result(result)
            check_zulip_realm_query_result(result)
            lear_nonreg_email = "newguy@lear.org"
            self.client_post("/accounts/home/", {"email": lear_nonreg_email}, subdomain="lear")
            result = get_check_query_result(lear_nonreg_email, 1)
            check_preregistration_user_query_result(result, lear_nonreg_email)
            check_lear_realm_query_result(result)
            self.login_user(lear_user)
            create_invitation("general", "newguy2@lear.org", lear_realm)
            result = get_check_query_result("newguy2@lear.org", 1, lear_realm.string_id)
            check_preregistration_user_query_result(result, "newguy2@lear.org", invite=True)
            check_lear_realm_query_result(result)
    def test_get_org_type_display_name(self) -> None:
        self.assertEqual(get_org_type_display_name(Realm.ORG_TYPES["business"]["id"]), "Business")
        self.assertEqual(get_org_type_display_name(883), "")
    def test_unspecified_org_type_correctly_displayed(self) -> None:
        """
        Unspecified org type is special in that it is marked to not be shown
        on the registration page (because organitions are not meant to be able to choose it),
        but should be correctly shown at the /support/ endpoint.
        """
        realm = get_realm("zulip")
        do_change_realm_org_type(realm, 0, acting_user=None)
        self.assertEqual(realm.org_type, 0)
        self.login("iago")
        result = self.client_get("/activity/support", {"q": "zulip"}, subdomain="zulip")
        self.assert_in_success_response(
            [
                f'<input type="hidden" name="realm_id" value="{realm.id}"',
                '<option value="0" selected>',
            ],
            result,
        )
    @mock.patch("analytics.views.support.update_billing_method_of_current_plan")
    def test_change_billing_method(self, m: mock.Mock) -> None:
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        result = self.client_post(
            "/activity/support",
            {"realm_id": f"{iago.realm_id}", "billing_method": "charge_automatically"},
        )
        m.assert_called_once_with(get_realm("zulip"), charge_automatically=True, acting_user=iago)
        self.assert_in_success_response(
            ["Billing method of zulip updated to charge automatically"], result
        )
        m.reset_mock()
        result = self.client_post(
            "/activity/support", {"realm_id": f"{iago.realm_id}", "billing_method": "send_invoice"}
        )
        m.assert_called_once_with(get_realm("zulip"), charge_automatically=False, acting_user=iago)
        self.assert_in_success_response(
            ["Billing method of zulip updated to pay by invoice"], result
        )
    def test_change_realm_plan_type(self) -> None:
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "2"}
            )
            m.assert_called_once_with(get_realm("zulip"), 2, acting_user=iago)
            self.assert_in_success_response(
                ["Plan type of zulip changed from self-hosted to limited"], result
            )
        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "10"}
            )
            m.assert_called_once_with(get_realm("zulip"), 10, acting_user=iago)
            self.assert_in_success_response(
                ["Plan type of zulip changed from self-hosted to plus"], result
            )
    def test_change_org_type(self) -> None:
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "org_type": "70"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        with mock.patch("analytics.views.support.do_change_realm_org_type") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{iago.realm_id}", "org_type": "70"}
            )
            m.assert_called_once_with(get_realm("zulip"), 70, acting_user=iago)
            self.assert_in_success_response(
                ["Org type of zulip changed from Business to Government"], result
            )
    def test_attach_discount(self) -> None:
        cordelia = self.example_user("cordelia")
        lear_realm = get_realm("lear")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login("iago")
        with mock.patch("analytics.views.support.attach_discount_to_realm") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
            )
            m.assert_called_once_with(get_realm("lear"), 25, acting_user=iago)
            self.assert_in_success_response(["Discount of lear changed to 25% from 0%"], result)
    def test_change_sponsorship_status(self) -> None:
        lear_realm = get_realm("lear")
        self.assertIsNone(get_customer_by_realm(lear_realm))
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
        )
        self.assert_in_success_response(["lear marked as pending sponsorship."], result)
        customer = get_customer_by_realm(lear_realm)
        assert customer is not None
        self.assertTrue(customer.sponsorship_pending)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "false"}
        )
        self.assert_in_success_response(["lear is no longer pending sponsorship."], result)
        customer = get_customer_by_realm(lear_realm)
        assert customer is not None
        self.assertFalse(customer.sponsorship_pending)
    def test_approve_sponsorship(self) -> None:
        lear_realm = get_realm("lear")
        update_sponsorship_status(lear_realm, True, acting_user=None)
        king_user = self.lear_user("king")
        king_user.role = UserProfile.ROLE_REALM_OWNER
        king_user.save()
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support",
            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        result = self.client_post(
            "/activity/support",
            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
        )
        self.assert_in_success_response(["Sponsorship approved for lear"], result)
        lear_realm.refresh_from_db()
        self.assertEqual(lear_realm.plan_type, Realm.PLAN_TYPE_STANDARD_FREE)
        customer = get_customer_by_realm(lear_realm)
        assert customer is not None
        self.assertFalse(customer.sponsorship_pending)
        messages = UserMessage.objects.filter(user_profile=king_user)
        self.assertIn(
            "request for sponsored hosting has been approved", messages[0].message.content
        )
        self.assert_length(messages, 1)
    def test_activate_or_deactivate_realm(self) -> None:
        cordelia = self.example_user("cordelia")
        lear_realm = get_realm("lear")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        self.login("iago")
        with mock.patch("analytics.views.support.do_deactivate_realm") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
            )
            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
            self.assert_in_success_response(["lear deactivated"], result)
        with mock.patch("analytics.views.support.do_send_realm_reactivation_email") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "active"}
            )
            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
            self.assert_in_success_response(
                ["Realm reactivation email sent to admins of lear"], result
            )
    def test_change_subdomain(self) -> None:
        cordelia = self.example_user("cordelia")
        lear_realm = get_realm("lear")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new_name"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        self.login("iago")
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/activity/support?q=new-name")
        realm_id = lear_realm.id
        lear_realm = get_realm("new-name")
        self.assertEqual(lear_realm.id, realm_id)
        self.assertTrue(Realm.objects.filter(string_id="lear").exists())
        self.assertTrue(Realm.objects.filter(string_id="lear")[0].deactivated)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
        )
        self.assert_in_success_response(
            ["Subdomain already in use. Please choose a different one."], result
        )
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "zulip"}
        )
        self.assert_in_success_response(
            ["Subdomain already in use. Please choose a different one."], result
        )
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "lear"}
        )
        self.assert_in_success_response(
            ["Subdomain already in use. Please choose a different one."], result
        )
        # Test renaming to a "reserved" subdomain
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "your-org"}
        )
        self.assert_in_success_response(
            ["Subdomain reserved. Please choose a different one."], result
        )
    def test_downgrade_realm(self) -> None:
        cordelia = self.example_user("cordelia")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        iago = self.example_user("iago")
        self.login_user(iago)
        with mock.patch("analytics.views.support.downgrade_at_the_end_of_billing_cycle") as m:
            result = self.client_post(
                "/activity/support",
                {
                    "realm_id": f"{iago.realm_id}",
                    "modify_plan": "downgrade_at_billing_cycle_end",
                },
            )
            m.assert_called_once_with(get_realm("zulip"))
            self.assert_in_success_response(
                ["zulip marked for downgrade at the end of billing cycle"], result
            )
        with mock.patch(
            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
        ) as m:
            result = self.client_post(
                "/activity/support",
                {
                    "realm_id": f"{iago.realm_id}",
                    "modify_plan": "downgrade_now_without_additional_licenses",
                },
            )
            m.assert_called_once_with(get_realm("zulip"))
            self.assert_in_success_response(
                ["zulip downgraded without creating additional invoices"], result
            )
        with mock.patch(
            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
        ) as m1:
            with mock.patch("analytics.views.support.void_all_open_invoices", return_value=1) as m2:
                result = self.client_post(
                    "/activity/support",
                    {
                        "realm_id": f"{iago.realm_id}",
                        "modify_plan": "downgrade_now_void_open_invoices",
                    },
                )
                m1.assert_called_once_with(get_realm("zulip"))
                m2.assert_called_once_with(get_realm("zulip"))
                self.assert_in_success_response(
                    ["zulip downgraded and voided 1 open invoices"], result
                )
        with mock.patch("analytics.views.support.switch_realm_from_standard_to_plus_plan") as m:
            result = self.client_post(
                "/activity/support",
                {
                    "realm_id": f"{iago.realm_id}",
                    "modify_plan": "upgrade_to_plus",
                },
            )
            m.assert_called_once_with(get_realm("zulip"))
            self.assert_in_success_response(["zulip upgraded to Plus"], result)
    def test_scrub_realm(self) -> None:
        cordelia = self.example_user("cordelia")
        lear_realm = get_realm("lear")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        self.login("iago")
        with mock.patch("analytics.views.support.do_scrub_realm") as m:
            result = self.client_post(
                "/activity/support", {"realm_id": f"{lear_realm.id}", "scrub_realm": "true"}
            )
            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
            self.assert_in_success_response(["lear scrubbed"], result)
        with mock.patch("analytics.views.support.do_scrub_realm") as m:
            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
            self.assert_json_error(result, "Invalid parameters")
            m.assert_not_called()
    def test_delete_user(self) -> None:
        cordelia = self.example_user("cordelia")
        hamlet = self.example_user("hamlet")
        hamlet_email = hamlet.delivery_email
        realm = get_realm("zulip")
        self.login_user(cordelia)
        result = self.client_post(
            "/activity/support", {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id}
        )
        self.assertEqual(result.status_code, 302)
        self.assertEqual(result["Location"], "/login/")
        self.login("iago")
        with mock.patch("analytics.views.support.do_delete_user_preserving_messages") as m:
            result = self.client_post(
                "/activity/support",
                {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id},
            )
            m.assert_called_once_with(hamlet)
            self.assert_in_success_response([f"{hamlet_email} in zulip deleted"], result)
--- a/analytics/urls.py
+++ b/analytics/urls.py
@@ -1,69 +1,9 @@
-from typing import List, Union
+from django.conf.urls import patterns, url
-from django.conf.urls import include
+i18n_urlpatterns = [
-from django.urls import path
+    url(r'^activity$', 'analytics.views.get_activity'),
-from django.urls.resolvers import URLPattern, URLResolver
+    url(r'^realm_activity/(?P<realm>[\S]+)/$', 'analytics.views.get_realm_activity'),
-
+    url(r'^user_activity/(?P<email>[\S]+)/$', 'analytics.views.get_user_activity'),
 from analytics.views.installation_activity import get_installation_activity
 from analytics.views.realm_activity import get_realm_activity
 from analytics.views.stats import (
    get_chart_data,
    get_chart_data_for_installation,
    get_chart_data_for_realm,
    get_chart_data_for_remote_installation,
    get_chart_data_for_remote_realm,
    stats,
    stats_for_installation,
    stats_for_realm,
    stats_for_remote_installation,
    stats_for_remote_realm,
 )
 from analytics.views.support import support
 from analytics.views.user_activity import get_user_activity
 from zerver.lib.rest import rest_path
 i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
    # Server admin (user_profile.is_staff) visible stats pages
    path("activity", get_installation_activity),
    path("activity/support", support, name="support"),
    path("realm_activity/<realm_str>/", get_realm_activity),
    path("user_activity/<user_profile_id>/", get_user_activity),
    path("stats/realm/<realm_str>/", stats_for_realm),
    path("stats/installation", stats_for_installation),
    path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
    path(
        "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
    ),
    # User-visible stats page
    path("stats", stats, name="stats"),
 ]
-# These endpoints are a part of the API (V1), which uses:
+urlpatterns = patterns('', *i18n_urlpatterns)
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
 # * Takes and returns json-formatted data
 #
 # See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
 #
 # All of these paths are accessed by either a /json or /api prefix
 v1_api_and_json_patterns = [
    # get data for the graphs at /stats
    rest_path("analytics/chart_data", GET=get_chart_data),
    rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
    rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
    rest_path(
        "analytics/chart_data/remote/<int:remote_server_id>/installation",
        GET=get_chart_data_for_remote_installation,
    ),
    rest_path(
        "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
        GET=get_chart_data_for_remote_realm,
    ),
 ]
 i18n_urlpatterns += [
    path("api/v1/", include(v1_api_and_json_patterns)),
    path("json/", include(v1_api_and_json_patterns)),
 ]
 urlpatterns = i18n_urlpatterns
--- a/analytics/views.py
+++ b/analytics/views.py
@@ -0,0 +1,925 @@
 from __future__ import absolute_import
 from __future__ import division
 from six import text_type
 from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union
 from django.db import connection
 from django.db.models.query import QuerySet
 from django.template import RequestContext, loader
 from django.core import urlresolvers
 from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
 from jinja2 import Markup as mark_safe
 from zerver.decorator import has_request_variables, REQ, zulip_internal
 from zerver.models import get_realm, UserActivity, UserActivityInterval, Realm
 from zerver.lib.timestamp import timestamp_to_datetime
 from collections import defaultdict
 from datetime import datetime, timedelta
 import itertools
 import time
 import re
 import pytz
 from six.moves import filter
 from six.moves import map
 from six.moves import range
 from six.moves import zip
 eastern_tz = pytz.timezone('US/Eastern')
 from zproject.jinja2 import render_to_response
 def make_table(title, cols, rows, has_row_class=False):
    # type: (str, List[str], List[Any], bool) -> str
    if not has_row_class:
        def fix_row(row):
            # type: (Any) -> Dict[str, Any]
            return dict(cells=row, row_class=None)
        rows = list(map(fix_row, rows))
    data = dict(title=title, cols=cols, rows=rows)
    content = loader.render_to_string(
        'analytics/ad_hoc_query.html',
        dict(data=data)
    )
    return content
 def dictfetchall(cursor):
    # type: (connection.cursor) -> List[Dict[str, Any]]
    "Returns all rows from a cursor as a dict"
    desc = cursor.description
    return [
        dict(list(zip([col[0] for col in desc], row)))
        for row in cursor.fetchall()
    ]
 def get_realm_day_counts():
    # type: () -> Dict[str, Dict[str, str]]
    query = '''
        select
            r.domain,
            (now()::date - pub_date::date) age,
            count(*) cnt
        from zerver_message m
        join zerver_userprofile up on up.id = m.sender_id
        join zerver_realm r on r.id = up.realm_id
        join zerver_client c on c.id = m.sending_client_id
        where
            (not up.is_bot)
        and
            pub_date > now()::date - interval '8 day'
        and
            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
        group by
            r.domain,
            age
        order by
            r.domain,
            age
    '''
    cursor = connection.cursor()
    cursor.execute(query)
    rows = dictfetchall(cursor)
    cursor.close()
    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
    for row in rows:
        counts[row['domain']][row['age']] = row['cnt']
    result = {}
    for domain in counts:
        raw_cnts = [counts[domain].get(age, 0) for age in range(8)]
        min_cnt = min(raw_cnts)
        max_cnt = max(raw_cnts)
        def format_count(cnt):
            # type: (int) -> str
            if cnt == min_cnt:
                good_bad = 'bad'
            elif cnt == max_cnt:
                good_bad = 'good'
            else:
                good_bad = 'neutral'
            return '<td class="number %s">%s</td>' % (good_bad, cnt)
        cnts = ''.join(map(format_count, raw_cnts))
        result[domain] = dict(cnts=cnts)
    return result
 def realm_summary_table(realm_minutes):
    # type: (Dict[str, float]) -> str
    query = '''
        SELECT
            realm.domain,
            coalesce(user_counts.active_user_count, 0) active_user_count,
            coalesce(at_risk_counts.at_risk_count, 0) at_risk_count,
            (
                SELECT
                    count(*)
                FROM zerver_userprofile up
                WHERE up.realm_id = realm.id
                AND is_active
                AND not is_bot
            ) user_profile_count,
            (
                SELECT
                    count(*)
                FROM zerver_userprofile up
                WHERE up.realm_id = realm.id
                AND is_active
                AND is_bot
            ) bot_count
        FROM zerver_realm realm
        LEFT OUTER JOIN
            (
                SELECT
                    up.realm_id realm_id,
                    count(distinct(ua.user_profile_id)) active_user_count
                FROM zerver_useractivity ua
                JOIN zerver_userprofile up
                    ON up.id = ua.user_profile_id
                WHERE
                    query in (
                        '/json/send_message',
                        'send_message_backend',
                        '/api/v1/send_message',
                        '/json/update_pointer',
                        '/json/users/me/pointer'
                    )
                AND
                    last_visit > now() - interval '1 day'
                AND
                    not is_bot
                GROUP BY realm_id
            ) user_counts
            ON user_counts.realm_id = realm.id
        LEFT OUTER JOIN
            (
                SELECT
                    realm_id,
                    count(*) at_risk_count
                FROM (
                    SELECT
                        realm.id as realm_id,
                        up.email
                    FROM zerver_useractivity ua
                    JOIN zerver_userprofile up
                        ON up.id = ua.user_profile_id
                    JOIN zerver_realm realm
                        ON realm.id = up.realm_id
                    WHERE up.is_active
                    AND (not up.is_bot)
                    AND
                        ua.query in (
                            '/json/send_message',
                            'send_message_backend',
                            '/api/v1/send_message',
                            '/json/update_pointer',
                            '/json/users/me/pointer'
                        )
                    GROUP by realm.id, up.email
                    HAVING max(last_visit) between
                        now() - interval '7 day' and
                        now() - interval '1 day'
                ) as at_risk_users
                GROUP BY realm_id
            ) at_risk_counts
            ON at_risk_counts.realm_id = realm.id
        WHERE EXISTS (
                SELECT *
                FROM zerver_useractivity ua
                JOIN zerver_userprofile up
                    ON up.id = ua.user_profile_id
                WHERE
                    query in (
                        '/json/send_message',
                        '/api/v1/send_message',
                        'send_message_backend',
                        '/json/update_pointer',
                        '/json/users/me/pointer'
                    )
                AND
                    up.realm_id = realm.id
                AND
                    last_visit > now() - interval '2 week'
        )
        ORDER BY active_user_count DESC, domain ASC
        '''
    cursor = connection.cursor()
    cursor.execute(query)
    rows = dictfetchall(cursor)
    cursor.close()
    # get messages sent per day
    counts = get_realm_day_counts()
    for row in rows:
        try:
            row['history'] = counts[row['domain']]['cnts']
        except:
            row['history'] = ''
    # augment data with realm_minutes
    total_hours = 0.0
    for row in rows:
        domain = row['domain']
        minutes = realm_minutes.get(domain, 0.0)
        hours = minutes / 60.0
        total_hours += hours
        row['hours'] = str(int(hours))
        try:
            row['hours_per_user'] = '%.1f' % (hours / row['active_user_count'],)
        except:
            pass
    # formatting
    for row in rows:
        row['domain'] = realm_activity_link(row['domain'])
    # Count active sites
    def meets_goal(row):
        # type: (Dict[str, int]) -> bool
        return row['active_user_count'] >= 5
    num_active_sites = len(list(filter(meets_goal, rows)))
    # create totals
    total_active_user_count = 0
    total_user_profile_count = 0
    total_bot_count = 0
    total_at_risk_count = 0
    for row in rows:
        total_active_user_count += int(row['active_user_count'])
        total_user_profile_count += int(row['user_profile_count'])
        total_bot_count += int(row['bot_count'])
        total_at_risk_count += int(row['at_risk_count'])
    rows.append(dict(
        domain='Total',
        active_user_count=total_active_user_count,
        user_profile_count=total_user_profile_count,
        bot_count=total_bot_count,
        hours=int(total_hours),
        at_risk_count=total_at_risk_count,
    ))
    content = loader.render_to_string(
        'analytics/realm_summary_table.html',
        dict(rows=rows, num_active_sites=num_active_sites)
    )
    return content
 def user_activity_intervals():
    # type: () -> Tuple[mark_safe, Dict[str, float]]
    day_end = timestamp_to_datetime(time.time())
    day_start = day_end - timedelta(hours=24)
    output = "Per-user online duration for the last 24 hours:\n"
    total_duration = timedelta(0)
    all_intervals = UserActivityInterval.objects.filter(
        end__gte=day_start,
        start__lte=day_end
    ).select_related(
        'user_profile',
        'user_profile__realm'
    ).only(
        'start',
        'end',
        'user_profile__email',
        'user_profile__realm__domain'
    ).order_by(
        'user_profile__realm__domain',
        'user_profile__email'
    )
    by_domain = lambda row: row.user_profile.realm.domain
    by_email = lambda row: row.user_profile.email
    realm_minutes = {}
    for domain, realm_intervals in itertools.groupby(all_intervals, by_domain):
        realm_duration = timedelta(0)
        output += '<hr>%s\n' % (domain,)
        for email, intervals in itertools.groupby(realm_intervals, by_email):
            duration = timedelta(0)
            for interval in intervals:
                start = max(day_start, interval.start)
                end = min(day_end, interval.end)
                duration += end - start
            total_duration += duration
            realm_duration += duration
            output += "  %-*s%s\n" % (37, email, duration)
        realm_minutes[domain] = realm_duration.total_seconds() / 60
    output += "\nTotal Duration:                      %s\n" % (total_duration,)
    output += "\nTotal Duration in minutes:           %s\n" % (total_duration.total_seconds() / 60.,)
    output += "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
    content = mark_safe('<pre>' + output + '</pre>')
    return content, realm_minutes
 def sent_messages_report(realm):
    # type: (str) -> str
    title = 'Recently sent messages for ' + realm
    cols = [
        'Date',
        'Humans',
        'Bots'
    ]
    query = '''
        select
            series.day::date,
            humans.cnt,
            bots.cnt
        from (
            select generate_series(
                (now()::date - interval '2 week'),
                now()::date,
                interval '1 day'
            ) as day
        ) as series
        left join (
            select
                pub_date::date pub_date,
                count(*) cnt
            from zerver_message m
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
                r.domain = %s
            and
                (not up.is_bot)
            and
                pub_date > now() - interval '2 week'
            group by
                pub_date::date
            order by
                pub_date::date
        ) humans on
            series.day = humans.pub_date
        left join (
            select
                pub_date::date pub_date,
                count(*) cnt
            from zerver_message m
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
                r.domain = %s
            and
                up.is_bot
            and
                pub_date > now() - interval '2 week'
            group by
                pub_date::date
            order by
                pub_date::date
        ) bots on
            series.day = bots.pub_date
    '''
    cursor = connection.cursor()
    cursor.execute(query, [realm, realm])
    rows = cursor.fetchall()
    cursor.close()
    return make_table(title, cols, rows)
 def ad_hoc_queries():
    # type: () -> List[Dict[str, str]]
    def get_page(query, cols, title):
        # type: (str, List[str], str) -> Dict[str, str]
        cursor = connection.cursor()
        cursor.execute(query)
        rows = cursor.fetchall()
        rows = list(map(list, rows))
        cursor.close()
        def fix_rows(i, fixup_func):
            # type: (int, Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None
            for row in rows:
                row[i] = fixup_func(row[i])
        for i, col in enumerate(cols):
            if col == 'Domain':
                fix_rows(i, realm_activity_link)
            elif col in ['Last time', 'Last visit']:
                fix_rows(i, format_date_for_activity_reports)
        content = make_table(title, cols, rows)
        return dict(
            content=content,
            title=title
        )
    pages = []
    ###
    for mobile_type in ['Android', 'ZulipiOS']:
        title = '%s usage' % (mobile_type,)
        query = '''
            select
                realm.domain,
                up.id user_id,
                client.name,
                sum(count) as hits,
                max(last_visit) as last_time
            from zerver_useractivity ua
            join zerver_client client on client.id = ua.client_id
            join zerver_userprofile up on up.id = ua.user_profile_id
            join zerver_realm realm on realm.id = up.realm_id
            where
                client.name like '%s'
            group by domain, up.id, client.name
            having max(last_visit) > now() - interval '2 week'
            order by domain, up.id, client.name
        ''' % (mobile_type,)
        cols = [
            'Domain',
            'User id',
            'Name',
            'Hits',
            'Last time'
        ]
        pages.append(get_page(query, cols, title))
    ###
    title = 'Desktop users'
    query = '''
        select
            realm.domain,
            client.name,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            client.name like 'desktop%%'
        group by domain, client.name
        having max(last_visit) > now() - interval '2 week'
        order by domain, client.name
    '''
    cols = [
        'Domain',
        'Client',
        'Hits',
        'Last time'
    ]
    pages.append(get_page(query, cols, title))
    ###
    title = 'Integrations by domain'
    query = '''
        select
            realm.domain,
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            (query in ('send_message_backend', '/api/v1/send_message')
            and client.name not in ('Android', 'ZulipiOS')
            and client.name not like 'test: Zulip%%'
            )
        or
            query like '%%external%%'
        group by domain, client_name
        having max(last_visit) > now() - interval '2 week'
        order by domain, client_name
    '''
    cols = [
        'Domain',
        'Client',
        'Hits',
        'Last time'
    ]
    pages.append(get_page(query, cols, title))
    ###
    title = 'Integrations by client'
    query = '''
        select
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
            realm.domain,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            (query in ('send_message_backend', '/api/v1/send_message')
            and client.name not in ('Android', 'ZulipiOS')
            and client.name not like 'test: Zulip%%'
            )
        or
            query like '%%external%%'
        group by client_name, domain
        having max(last_visit) > now() - interval '2 week'
        order by client_name, domain
    '''
    cols = [
        'Client',
        'Domain',
        'Hits',
        'Last time'
    ]
    pages.append(get_page(query, cols, title))
    return pages
@zulip_internal
@has_request_variables
 def get_activity(request):
    # type: (HttpRequest) -> HttpResponse
    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
    counts_content = realm_summary_table(realm_minutes) # type: str
    data = [
        ('Counts', counts_content),
        ('Durations', duration_content),
    ]
    for page in ad_hoc_queries():
        data.append((page['title'], page['content']))
    title = 'Activity'
    return render_to_response(
        'analytics/activity.html',
        dict(data=data, title=title, is_home=True),
        request=request
    )
 def get_user_activity_records_for_realm(realm, is_bot):
    # type: (str, bool) -> QuerySet
    fields = [
        'user_profile__full_name',
        'user_profile__email',
        'query',
        'client__name',
        'count',
        'last_visit',
    ]
    records = UserActivity.objects.filter(
            user_profile__realm__domain=realm,
            user_profile__is_active=True,
            user_profile__is_bot=is_bot
    )
    records = records.order_by("user_profile__email", "-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
    return records
 def get_user_activity_records_for_email(email):
    # type: (str) -> List[QuerySet]
    fields = [
        'user_profile__full_name',
        'query',
        'client__name',
        'count',
        'last_visit'
    ]
    records = UserActivity.objects.filter(
            user_profile__email=email
    )
    records = records.order_by("-last_visit")
    records = records.select_related('user_profile', 'client').only(*fields)
    return records
 def raw_user_activity_table(records):
    # type: (List[QuerySet]) -> str
    cols = [
        'query',
        'client',
        'count',
        'last_visit'
    ]
    def row(record):
        # type: (QuerySet) -> List[Any]
        return [
                record.query,
                record.client.name,
                record.count,
                format_date_for_activity_reports(record.last_visit)
        ]
    rows = list(map(row, records))
    title = 'Raw Data'
    return make_table(title, cols, rows)
 def get_user_activity_summary(records):
    # type: (List[QuerySet]) -> Dict[str, Dict[str, Any]]
    #: `Any` used above should be `Union(int, datetime)`.
    #: However current version of `Union` does not work inside other function.
    #: We could use something like:
    # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
    #: but that would require this long `Union` to carry on throughout inner functions.
    summary = {} # type: Dict[str, Dict[str, Any]]
    def update(action, record):
        # type: (str, QuerySet) -> None
        if action not in summary:
            summary[action] = dict(
                    count=record.count,
                    last_visit=record.last_visit
            )
        else:
            summary[action]['count'] += record.count
            summary[action]['last_visit'] = max(
                    summary[action]['last_visit'],
                    record.last_visit
            )
    if records:
        summary['name'] = records[0].user_profile.full_name
    for record in records:
        client = record.client.name
        query = record.query
        update('use', record)
        if client == 'API':
            m = re.match('/api/.*/external/(.*)', query)
            if m:
                client = m.group(1)
                update(client, record)
        if client.startswith('desktop'):
            update('desktop', record)
        if client == 'website':
            update('website', record)
        if ('send_message' in query) or re.search('/api/.*/external/.*', query):
            update('send', record)
        if query in ['/json/update_pointer', '/json/users/me/pointer', '/api/v1/update_pointer']:
            update('pointer', record)
        update(client, record)
    return summary
 def format_date_for_activity_reports(date):
    # type: (Optional[datetime]) -> str
    if date:
        return date.astimezone(eastern_tz).strftime('%Y-%m-%d %H:%M')
    else:
        return ''
 def user_activity_link(email):
    # type: (str) -> mark_safe
    url_name = 'analytics.views.get_user_activity'
    url = urlresolvers.reverse(url_name, kwargs=dict(email=email))
    email_link = '<a href="%s">%s</a>' % (url, email)
    return mark_safe(email_link)
 def realm_activity_link(realm):
    # type: (str) -> mark_safe
    url_name = 'analytics.views.get_realm_activity'
    url = urlresolvers.reverse(url_name, kwargs=dict(realm=realm))
    realm_link = '<a href="%s">%s</a>' % (url, realm)
    return mark_safe(realm_link)
 def realm_client_table(user_summaries):
    # type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
    exclude_keys = [
            'internal',
            'name',
            'use',
            'send',
            'pointer',
            'website',
            'desktop',
    ]
    rows = []
    for email, user_summary in user_summaries.items():
        email_link = user_activity_link(email)
        name = user_summary['name']
        for k, v in user_summary.items():
            if k in exclude_keys:
                continue
            client = k
            count = v['count']
            last_visit = v['last_visit']
            row = [
                    format_date_for_activity_reports(last_visit),
                    client,
                    name,
                    email_link,
                    count,
            ]
            rows.append(row)
    rows = sorted(rows, key=lambda r: r[0], reverse=True)
    cols = [
            'Last visit',
            'Client',
            'Name',
            'Email',
            'Count',
    ]
    title = 'Clients'
    return make_table(title, cols, rows)
 def user_activity_summary_table(user_summary):
    # type: (Dict[str, Dict[str, Any]]) -> str
    rows = []
    for k, v in user_summary.items():
        if k == 'name':
            continue
        client = k
        count = v['count']
        last_visit = v['last_visit']
        row = [
                format_date_for_activity_reports(last_visit),
                client,
                count,
        ]
        rows.append(row)
    rows = sorted(rows, key=lambda r: r[0], reverse=True)
    cols = [
            'last_visit',
            'client',
            'count',
    ]
    title = 'User Activity'
    return make_table(title, cols, rows)
 def realm_user_summary_table(all_records, admin_emails):
    # type: (List[QuerySet], Set[text_type]) -> Tuple[Dict[str, Dict[str, Any]], str]
    user_records = {}
    def by_email(record):
        # type: (QuerySet) -> str
        return record.user_profile.email
    for email, records in itertools.groupby(all_records, by_email):
        user_records[email] = get_user_activity_summary(list(records))
    def get_last_visit(user_summary, k):
        # type: (Dict[str, Dict[str, datetime]], str) -> Optional[datetime]
        if k in user_summary:
            return user_summary[k]['last_visit']
        else:
            return None
    def get_count(user_summary, k):
        # type: (Dict[str, Dict[str, str]], str) -> str
        if k in user_summary:
            return user_summary[k]['count']
        else:
            return ''
    def is_recent(val):
        # type: (Optional[datetime]) -> bool
        age = datetime.now(val.tzinfo) - val # type: ignore # datetie.now tzinfo bug.
        return age.total_seconds() < 5 * 60
    rows = []
    for email, user_summary in user_records.items():
        email_link = user_activity_link(email)
        sent_count = get_count(user_summary, 'send')
        cells = [user_summary['name'], email_link, sent_count]
        row_class = ''
        for field in ['use', 'send', 'pointer', 'desktop', 'ZulipiOS', 'Android']:
            visit = get_last_visit(user_summary, field)
            if field == 'use':
                if visit and is_recent(visit):
                    row_class += ' recently_active'
                if email in admin_emails:
                    row_class += ' admin'
            val = format_date_for_activity_reports(visit)
            cells.append(val)
        row = dict(cells=cells, row_class=row_class)
        rows.append(row)
    def by_used_time(row):
        # type: (Dict[str, Sequence[str]]) -> str
        return row['cells'][3]
    rows = sorted(rows, key=by_used_time, reverse=True)
    cols = [
            'Name',
            'Email',
            'Total sent',
            'Heard from',
            'Message sent',
            'Pointer motion',
            'Desktop',
            'ZulipiOS',
            'Android'
    ]
    title = 'Summary'
    content = make_table(title, cols, rows, has_row_class=True)
    return user_records, content
@zulip_internal
 def get_realm_activity(request, realm):
    # type: (HttpRequest, str) -> HttpResponse
    data = [] # type: List[Tuple[str, str]]
    all_user_records = {} # type: Dict[str, Any]
    try:
        admins = get_realm(realm).get_admin_users()
    except Realm.DoesNotExist:
        return HttpResponseNotFound("Realm %s does not exist" % (realm,))
    admin_emails = {admin.email for admin in admins}
    for is_bot, page_title in [(False,  'Humans'), (True, 'Bots')]:
        all_records = list(get_user_activity_records_for_realm(realm, is_bot))
        user_records, content = realm_user_summary_table(all_records, admin_emails)
        all_user_records.update(user_records)
        data += [(page_title, content)]
    page_title = 'Clients'
    content = realm_client_table(all_user_records)
    data += [(page_title, content)]
    page_title = 'History'
    content = sent_messages_report(realm)
    data += [(page_title, content)]
    fix_name = lambda realm: realm.replace('.', '_')
    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (fix_name(realm),)
    title = realm
    return render_to_response(
        'analytics/activity.html',
        dict(data=data, realm_link=realm_link, title=title),
        request=request
    )
@zulip_internal
 def get_user_activity(request, email):
    # type: (HttpRequest, str) -> HttpResponse
    records = get_user_activity_records_for_email(email)
    data = [] # type: List[Tuple[str, str]]
    user_summary = get_user_activity_summary(records)
    content = user_activity_summary_table(user_summary)
    data += [('Summary', content)]
    content = raw_user_activity_table(records)
    data += [('Info', content)]
    title = email
    return render_to_response(
        'analytics/activity.html',
        dict(data=data, title=title),
        request=request
    )
--- a/analytics/views/activity_common.py
+++ b/analytics/views/activity_common.py
@@ -1,154 +0,0 @@
 import re
 import sys
 from datetime import datetime
 from typing import Any, Collection, Dict, List, Optional, Sequence
 from urllib.parse import urlencode
 from django.conf import settings
 from django.db.backends.utils import CursorWrapper
 from django.template import loader
 from django.urls import reverse
 from markupsafe import Markup
 from zerver.lib.url_encoding import append_url_query_string
 from zerver.models import UserActivity, get_realm
 if sys.version_info < (3, 9):  # nocoverage
    from backports import zoneinfo
 else:  # nocoverage
    import zoneinfo
 eastern_tz = zoneinfo.ZoneInfo("America/New_York")
 if settings.BILLING_ENABLED:
    pass
 def make_table(
    title: str, cols: Sequence[str], rows: Sequence[Any], has_row_class: bool = False
 ) -> str:
    if not has_row_class:
        def fix_row(row: Any) -> Dict[str, Any]:
            return dict(cells=row, row_class=None)
        rows = list(map(fix_row, rows))
    data = dict(title=title, cols=cols, rows=rows)
    content = loader.render_to_string(
        "analytics/ad_hoc_query.html",
        dict(data=data),
    )
    return content
 def dictfetchall(cursor: CursorWrapper) -> List[Dict[str, Any]]:
    """Returns all rows from a cursor as a dict"""
    desc = cursor.description
    return [dict(zip((col[0] for col in desc), row)) for row in cursor.fetchall()]
 def format_date_for_activity_reports(date: Optional[datetime]) -> str:
    if date:
        return date.astimezone(eastern_tz).strftime("%Y-%m-%d %H:%M")
    else:
        return ""
 def user_activity_link(email: str, user_profile_id: int) -> Markup:
    from analytics.views.user_activity import get_user_activity
    url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
 def realm_activity_link(realm_str: str) -> Markup:
    from analytics.views.realm_activity import get_realm_activity
    url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
 def realm_stats_link(realm_str: str) -> Markup:
    from analytics.views.stats import stats_for_realm
    url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
 def realm_support_link(realm_str: str) -> Markup:
    support_url = reverse("support")
    query = urlencode({"q": realm_str})
    url = append_url_query_string(support_url, query)
    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
 def realm_url_link(realm_str: str) -> Markup:
    url = get_realm(realm_str).uri
    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
 def remote_installation_stats_link(server_id: int, hostname: str) -> Markup:
    from analytics.views.stats import stats_for_remote_installation
    url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i>{hostname}</a>').format(
        url=url, hostname=hostname
    )
 def get_user_activity_summary(records: Collection[UserActivity]) -> Dict[str, Any]:
    #: The type annotation used above is clearly overly permissive.
    #: We should perhaps use TypedDict to clearly lay out the schema
    #: for the user activity summary.
    summary: Dict[str, Any] = {}
    def update(action: str, record: UserActivity) -> None:
        if action not in summary:
            summary[action] = dict(
                count=record.count,
                last_visit=record.last_visit,
            )
        else:
            summary[action]["count"] += record.count
            summary[action]["last_visit"] = max(
                summary[action]["last_visit"],
                record.last_visit,
            )
    if records:
        first_record = next(iter(records))
        summary["name"] = first_record.user_profile.full_name
        summary["user_profile_id"] = first_record.user_profile.id
    for record in records:
        client = record.client.name
        query = str(record.query)
        update("use", record)
        if client == "API":
            m = re.match("/api/.*/external/(.*)", query)
            if m:
                client = m.group(1)
                update(client, record)
        if client.startswith("desktop"):
            update("desktop", record)
        if client == "website":
            update("website", record)
        if ("send_message" in query) or re.search("/api/.*/external/.*", query):
            update("send", record)
        if query in [
            "/json/update_pointer",
            "/json/users/me/pointer",
            "/api/v1/update_pointer",
            "update_pointer_backend",
        ]:
            update("pointer", record)
        update(client, record)
    return summary
--- a/analytics/views/installation_activity.py
+++ b/analytics/views/installation_activity.py
@@ -1,620 +0,0 @@
 import itertools
 import time
 from collections import defaultdict
 from contextlib import suppress
 from datetime import datetime, timedelta
 from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 from django.conf import settings
 from django.db import connection
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 from django.template import loader
 from django.utils.timezone import now as timezone_now
 from markupsafe import Markup
 from psycopg2.sql import SQL, Composable, Literal
 from analytics.lib.counts import COUNT_STATS
 from analytics.views.activity_common import (
    dictfetchall,
    format_date_for_activity_reports,
    make_table,
    realm_activity_link,
    realm_stats_link,
    realm_support_link,
    realm_url_link,
    remote_installation_stats_link,
 )
 from analytics.views.support import get_plan_name
 from zerver.decorator import require_server_admin
 from zerver.lib.request import has_request_variables
 from zerver.lib.timestamp import timestamp_to_datetime
 from zerver.models import Realm, UserActivityInterval, get_org_type_display_name
 if settings.BILLING_ENABLED:
    from corporate.lib.stripe import (
        estimate_annual_recurring_revenue_by_realm,
        get_realms_to_default_discount_dict,
    )
 def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
    query = SQL(
        """
        select
            r.string_id,
            (now()::date - date_sent::date) age,
            count(*) cnt
        from zerver_message m
        join zerver_userprofile up on up.id = m.sender_id
        join zerver_realm r on r.id = up.realm_id
        join zerver_client c on c.id = m.sending_client_id
        where
            (not up.is_bot)
        and
            date_sent > now()::date - interval '8 day'
        and
            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
        group by
            r.string_id,
            age
        order by
            r.string_id,
            age
    """
    )
    cursor = connection.cursor()
    cursor.execute(query)
    rows = dictfetchall(cursor)
    cursor.close()
    counts: Dict[str, Dict[int, int]] = defaultdict(dict)
    for row in rows:
        counts[row["string_id"]][row["age"]] = row["cnt"]
    result = {}
    for string_id in counts:
        raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
        min_cnt = min(raw_cnts[1:])
        max_cnt = max(raw_cnts[1:])
        def format_count(cnt: int, style: Optional[str] = None) -> Markup:
            if style is not None:
                good_bad = style
            elif cnt == min_cnt:
                good_bad = "bad"
            elif cnt == max_cnt:
                good_bad = "good"
            else:
                good_bad = "neutral"
            return Markup('<td class="number {good_bad}">{cnt}</td>').format(
                good_bad=good_bad, cnt=cnt
            )
        cnts = format_count(raw_cnts[0], "neutral") + Markup().join(map(format_count, raw_cnts[1:]))
        result[string_id] = dict(cnts=cnts)
    return result
 def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
    now = timezone_now()
    query = SQL(
        """
        SELECT
            realm.string_id,
            realm.date_created,
            realm.plan_type,
            realm.org_type,
            coalesce(wau_table.value, 0) wau_count,
            coalesce(dau_table.value, 0) dau_count,
            coalesce(user_count_table.value, 0) user_profile_count,
            coalesce(bot_count_table.value, 0) bot_count
        FROM
            zerver_realm as realm
            LEFT OUTER JOIN (
                SELECT
                    value _14day_active_humans,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'realm_active_humans::day'
                    AND end_time = %(realm_active_humans_end_time)s
            ) as _14day_active_humans_table ON realm.id = _14day_active_humans_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = '7day_actives::day'
                    AND end_time = %(seven_day_actives_end_time)s
            ) as wau_table ON realm.id = wau_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = '1day_actives::day'
                    AND end_time = %(one_day_actives_end_time)s
            ) as dau_table ON realm.id = dau_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'active_users_audit:is_bot:day'
                    AND subgroup = 'false'
                    AND end_time = %(active_users_audit_end_time)s
            ) as user_count_table ON realm.id = user_count_table.realm_id
            LEFT OUTER JOIN (
                SELECT
                    value,
                    realm_id
                from
                    analytics_realmcount
                WHERE
                    property = 'active_users_audit:is_bot:day'
                    AND subgroup = 'true'
                    AND end_time = %(active_users_audit_end_time)s
            ) as bot_count_table ON realm.id = bot_count_table.realm_id
        WHERE
            _14day_active_humans IS NOT NULL
            or realm.plan_type = 3
        ORDER BY
            dau_count DESC,
            string_id ASC
    """
    )
    cursor = connection.cursor()
    cursor.execute(
        query,
        {
            "realm_active_humans_end_time": COUNT_STATS[
                "realm_active_humans::day"
            ].last_successful_fill(),
            "seven_day_actives_end_time": COUNT_STATS["7day_actives::day"].last_successful_fill(),
            "one_day_actives_end_time": COUNT_STATS["1day_actives::day"].last_successful_fill(),
            "active_users_audit_end_time": COUNT_STATS[
                "active_users_audit:is_bot:day"
            ].last_successful_fill(),
        },
    )
    rows = dictfetchall(cursor)
    cursor.close()
    for row in rows:
        row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
        row["age_days"] = int((now - row["date_created"]).total_seconds() / 86400)
        row["is_new"] = row["age_days"] < 12 * 7
    # get messages sent per day
    counts = get_realm_day_counts()
    for row in rows:
        try:
            row["history"] = counts[row["string_id"]]["cnts"]
        except Exception:
            row["history"] = ""
    # estimate annual subscription revenue
    total_arr = 0
    if settings.BILLING_ENABLED:
        estimated_arrs = estimate_annual_recurring_revenue_by_realm()
        realms_to_default_discount = get_realms_to_default_discount_dict()
        for row in rows:
            row["plan_type_string"] = get_plan_name(row["plan_type"])
            string_id = row["string_id"]
            if string_id in estimated_arrs:
                row["arr"] = estimated_arrs[string_id]
            if row["plan_type"] in [Realm.PLAN_TYPE_STANDARD, Realm.PLAN_TYPE_PLUS]:
                row["effective_rate"] = 100 - int(realms_to_default_discount.get(string_id, 0))
            elif row["plan_type"] == Realm.PLAN_TYPE_STANDARD_FREE:
                row["effective_rate"] = 0
            elif (
                row["plan_type"] == Realm.PLAN_TYPE_LIMITED
                and string_id in realms_to_default_discount
            ):
                row["effective_rate"] = 100 - int(realms_to_default_discount[string_id])
            else:
                row["effective_rate"] = ""
        total_arr += sum(estimated_arrs.values())
    for row in rows:
        row["org_type_string"] = get_org_type_display_name(row["org_type"])
    # augment data with realm_minutes
    total_hours = 0.0
    for row in rows:
        string_id = row["string_id"]
        minutes = realm_minutes.get(string_id, 0.0)
        hours = minutes / 60.0
        total_hours += hours
        row["hours"] = str(int(hours))
        with suppress(Exception):
            row["hours_per_user"] = "{:.1f}".format(hours / row["dau_count"])
    # formatting
    for row in rows:
        row["realm_url"] = realm_url_link(row["string_id"])
        row["stats_link"] = realm_stats_link(row["string_id"])
        row["support_link"] = realm_support_link(row["string_id"])
        row["string_id"] = realm_activity_link(row["string_id"])
    # Count active sites
    def meets_goal(row: Dict[str, int]) -> bool:
        return row["dau_count"] >= 5
    num_active_sites = len(list(filter(meets_goal, rows)))
    # create totals
    total_dau_count = 0
    total_user_profile_count = 0
    total_bot_count = 0
    total_wau_count = 0
    for row in rows:
        total_dau_count += int(row["dau_count"])
        total_user_profile_count += int(row["user_profile_count"])
        total_bot_count += int(row["bot_count"])
        total_wau_count += int(row["wau_count"])
    total_row = dict(
        string_id="Total",
        plan_type_string="",
        org_type_string="",
        effective_rate="",
        arr=total_arr,
        realm_url="",
        stats_link="",
        support_link="",
        date_created_day="",
        dau_count=total_dau_count,
        user_profile_count=total_user_profile_count,
        bot_count=total_bot_count,
        hours=int(total_hours),
        wau_count=total_wau_count,
    )
    rows.insert(0, total_row)
    content = loader.render_to_string(
        "analytics/realm_summary_table.html",
        dict(
            rows=rows,
            num_active_sites=num_active_sites,
            utctime=now.strftime("%Y-%m-%d %H:%M %Z"),
            billing_enabled=settings.BILLING_ENABLED,
        ),
    )
    return content
 def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
    day_end = timestamp_to_datetime(time.time())
    day_start = day_end - timedelta(hours=24)
    output = Markup()
    output += "Per-user online duration for the last 24 hours:\n"
    total_duration = timedelta(0)
    all_intervals = (
        UserActivityInterval.objects.filter(
            end__gte=day_start,
            start__lte=day_end,
        )
        .select_related(
            "user_profile",
            "user_profile__realm",
        )
        .only(
            "start",
            "end",
            "user_profile__delivery_email",
            "user_profile__realm__string_id",
        )
        .order_by(
            "user_profile__realm__string_id",
            "user_profile__delivery_email",
        )
    )
    by_string_id = lambda row: row.user_profile.realm.string_id
    by_email = lambda row: row.user_profile.delivery_email
    realm_minutes = {}
    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
        realm_duration = timedelta(0)
        output += Markup("<hr>") + f"{string_id}\n"
        for email, intervals in itertools.groupby(realm_intervals, by_email):
            duration = timedelta(0)
            for interval in intervals:
                start = max(day_start, interval.start)
                end = min(day_end, interval.end)
                duration += end - start
            total_duration += duration
            realm_duration += duration
            output += f"  {email:<37}{duration}\n"
        realm_minutes[string_id] = realm_duration.total_seconds() / 60
    output += f"\nTotal duration:                      {total_duration}\n"
    output += f"\nTotal duration in minutes:           {total_duration.total_seconds() / 60.}\n"
    output += f"Total duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}"
    content = Markup("<pre>{}</pre>").format(output)
    return content, realm_minutes
 def ad_hoc_queries() -> List[Dict[str, str]]:
    def get_page(
        query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
    ) -> Dict[str, str]:
        cursor = connection.cursor()
        cursor.execute(query)
        rows = cursor.fetchall()
        rows = list(map(list, rows))
        cursor.close()
        def fix_rows(
            i: int, fixup_func: Union[Callable[[str], Markup], Callable[[datetime], str]]
        ) -> None:
            for row in rows:
                row[i] = fixup_func(row[i])
        total_row = []
        for i, col in enumerate(cols):
            if col == "Realm":
                fix_rows(i, realm_activity_link)
            elif col in ["Last time", "Last visit"]:
                fix_rows(i, format_date_for_activity_reports)
            elif col == "Hostname":
                for row in rows:
                    row[i] = remote_installation_stats_link(row[0], row[i])
            if len(totals_columns) > 0:
                if i == 0:
                    total_row.append("Total")
                elif i in totals_columns:
                    total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
                else:
                    total_row.append("")
        if len(totals_columns) > 0:
            rows.insert(0, total_row)
        content = make_table(title, cols, rows)
        return dict(
            content=content,
            title=title,
        )
    pages = []
    ###
    for mobile_type in ["Android", "ZulipiOS"]:
        title = f"{mobile_type} usage"
        query: Composable = SQL(
            """
            select
                realm.string_id,
                up.id user_id,
                client.name,
                sum(count) as hits,
                max(last_visit) as last_time
            from zerver_useractivity ua
            join zerver_client client on client.id = ua.client_id
            join zerver_userprofile up on up.id = ua.user_profile_id
            join zerver_realm realm on realm.id = up.realm_id
            where
                client.name like {mobile_type}
            group by string_id, up.id, client.name
            having max(last_visit) > now() - interval '2 week'
            order by string_id, up.id, client.name
        """
        ).format(
            mobile_type=Literal(mobile_type),
        )
        cols = [
            "Realm",
            "User id",
            "Name",
            "Hits",
            "Last time",
        ]
        pages.append(get_page(query, cols, title))
    ###
    title = "Desktop users"
    query = SQL(
        """
        select
            realm.string_id,
            client.name,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            client.name like 'desktop%%'
        group by string_id, client.name
        having max(last_visit) > now() - interval '2 week'
        order by string_id, client.name
    """
    )
    cols = [
        "Realm",
        "Client",
        "Hits",
        "Last time",
    ]
    pages.append(get_page(query, cols, title))
    ###
    title = "Integrations by realm"
    query = SQL(
        """
        select
            realm.string_id,
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            (query in ('send_message_backend', '/api/v1/send_message')
            and client.name not in ('Android', 'ZulipiOS')
            and client.name not like 'test: Zulip%%'
            )
        or
            query like '%%external%%'
        group by string_id, client_name
        having max(last_visit) > now() - interval '2 week'
        order by string_id, client_name
    """
    )
    cols = [
        "Realm",
        "Client",
        "Hits",
        "Last time",
    ]
    pages.append(get_page(query, cols, title))
    ###
    title = "Integrations by client"
    query = SQL(
        """
        select
            case
                when query like '%%external%%' then split_part(query, '/', 5)
                else client.name
            end client_name,
            realm.string_id,
            sum(count) as hits,
            max(last_visit) as last_time
        from zerver_useractivity ua
        join zerver_client client on client.id = ua.client_id
        join zerver_userprofile up on up.id = ua.user_profile_id
        join zerver_realm realm on realm.id = up.realm_id
        where
            (query in ('send_message_backend', '/api/v1/send_message')
            and client.name not in ('Android', 'ZulipiOS')
            and client.name not like 'test: Zulip%%'
            )
        or
            query like '%%external%%'
        group by client_name, string_id
        having max(last_visit) > now() - interval '2 week'
        order by client_name, string_id
    """
    )
    cols = [
        "Client",
        "Realm",
        "Hits",
        "Last time",
    ]
    pages.append(get_page(query, cols, title))
    title = "Remote Zulip servers"
    query = SQL(
        """
        with icount as (
            select
                server_id,
                max(value) as max_value,
                max(end_time) as max_end_time
            from zilencer_remoteinstallationcount
            where
                property='active_users:is_bot:day'
                and subgroup='false'
            group by server_id
            ),
        remote_push_devices as (
            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
            group by server_id
        )
        select
            rserver.id,
            rserver.hostname,
            rserver.contact_email,
            max_value,
            push_user_count,
            max_end_time
        from zilencer_remotezulipserver rserver
        left join icount on icount.server_id = rserver.id
        left join remote_push_devices on remote_push_devices.server_id = rserver.id
        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
    """
    )
    cols = [
        "ID",
        "Hostname",
        "Contact email",
        "Analytics users",
        "Mobile users",
        "Last update time",
    ]
    pages.append(get_page(query, cols, title, totals_columns=[3, 4]))
    return pages
@require_server_admin
@has_request_variables
 def get_installation_activity(request: HttpRequest) -> HttpResponse:
    duration_content, realm_minutes = user_activity_intervals()
    counts_content: str = realm_summary_table(realm_minutes)
    data = [
        ("Counts", counts_content),
        ("Durations", duration_content),
    ]
    for page in ad_hoc_queries():
        data.append((page["title"], page["content"]))
    title = "Activity"
    return render(
        request,
        "analytics/activity.html",
        context=dict(data=data, title=title, is_home=True),
    )
--- a/analytics/views/realm_activity.py
+++ b/analytics/views/realm_activity.py
@@ -1,262 +0,0 @@
 import itertools
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Set, Tuple
 from django.db import connection
 from django.db.models import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils.timezone import now as timezone_now
 from psycopg2.sql import SQL
 from analytics.views.activity_common import (
    format_date_for_activity_reports,
    get_user_activity_summary,
    make_table,
    realm_stats_link,
    user_activity_link,
 )
 from zerver.decorator import require_server_admin
 from zerver.models import Realm, UserActivity
 def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet[UserActivity]:
    fields = [
        "user_profile__full_name",
        "user_profile__delivery_email",
        "query",
        "client__name",
        "count",
        "last_visit",
    ]
    records = UserActivity.objects.filter(
        user_profile__realm__string_id=realm,
        user_profile__is_active=True,
        user_profile__is_bot=is_bot,
    )
    records = records.order_by("user_profile__delivery_email", "-last_visit")
    records = records.select_related("user_profile", "client").only(*fields)
    return records
 def realm_user_summary_table(
    all_records: QuerySet[UserActivity], admin_emails: Set[str]
 ) -> Tuple[Dict[str, Any], str]:
    user_records = {}
    def by_email(record: UserActivity) -> str:
        return record.user_profile.delivery_email
    for email, records in itertools.groupby(all_records, by_email):
        user_records[email] = get_user_activity_summary(list(records))
    def get_last_visit(user_summary: Dict[str, Dict[str, datetime]], k: str) -> Optional[datetime]:
        if k in user_summary:
            return user_summary[k]["last_visit"]
        else:
            return None
    def get_count(user_summary: Dict[str, Dict[str, str]], k: str) -> str:
        if k in user_summary:
            return user_summary[k]["count"]
        else:
            return ""
    def is_recent(val: datetime) -> bool:
        age = timezone_now() - val
        return age.total_seconds() < 5 * 60
    rows = []
    for email, user_summary in user_records.items():
        email_link = user_activity_link(email, user_summary["user_profile_id"])
        sent_count = get_count(user_summary, "send")
        cells = [user_summary["name"], email_link, sent_count]
        row_class = ""
        for field in ["use", "send", "pointer", "desktop", "ZulipiOS", "Android"]:
            visit = get_last_visit(user_summary, field)
            if field == "use":
                if visit and is_recent(visit):
                    row_class += " recently_active"
                if email in admin_emails:
                    row_class += " admin"
            val = format_date_for_activity_reports(visit)
            cells.append(val)
        row = dict(cells=cells, row_class=row_class)
        rows.append(row)
    def by_used_time(row: Dict[str, Any]) -> str:
        return row["cells"][3]
    rows = sorted(rows, key=by_used_time, reverse=True)
    cols = [
        "Name",
        "Email",
        "Total sent",
        "Heard from",
        "Message sent",
        "Pointer motion",
        "Desktop",
        "ZulipiOS",
        "Android",
    ]
    title = "Summary"
    content = make_table(title, cols, rows, has_row_class=True)
    return user_records, content
 def realm_client_table(user_summaries: Dict[str, Dict[str, Any]]) -> str:
    exclude_keys = [
        "internal",
        "name",
        "user_profile_id",
        "use",
        "send",
        "pointer",
        "website",
        "desktop",
    ]
    rows = []
    for email, user_summary in user_summaries.items():
        email_link = user_activity_link(email, user_summary["user_profile_id"])
        name = user_summary["name"]
        for k, v in user_summary.items():
            if k in exclude_keys:
                continue
            client = k
            count = v["count"]
            last_visit = v["last_visit"]
            row = [
                format_date_for_activity_reports(last_visit),
                client,
                name,
                email_link,
                count,
            ]
            rows.append(row)
    rows = sorted(rows, key=lambda r: r[0], reverse=True)
    cols = [
        "Last visit",
        "Client",
        "Name",
        "Email",
        "Count",
    ]
    title = "Clients"
    return make_table(title, cols, rows)
 def sent_messages_report(realm: str) -> str:
    title = "Recently sent messages for " + realm
    cols = [
        "Date",
        "Humans",
        "Bots",
    ]
    query = SQL(
        """
        select
            series.day::date,
            humans.cnt,
            bots.cnt
        from (
            select generate_series(
                (now()::date - interval '2 week'),
                now()::date,
                interval '1 day'
            ) as day
        ) as series
        left join (
            select
                date_sent::date date_sent,
                count(*) cnt
            from zerver_message m
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
                r.string_id = %s
            and
                (not up.is_bot)
            and
                date_sent > now() - interval '2 week'
            group by
                date_sent::date
            order by
                date_sent::date
        ) humans on
            series.day = humans.date_sent
        left join (
            select
                date_sent::date date_sent,
                count(*) cnt
            from zerver_message m
            join zerver_userprofile up on up.id = m.sender_id
            join zerver_realm r on r.id = up.realm_id
            where
                r.string_id = %s
            and
                up.is_bot
            and
                date_sent > now() - interval '2 week'
            group by
                date_sent::date
            order by
                date_sent::date
        ) bots on
            series.day = bots.date_sent
    """
    )
    cursor = connection.cursor()
    cursor.execute(query, [realm, realm])
    rows = cursor.fetchall()
    cursor.close()
    return make_table(title, cols, rows)
@require_server_admin
 def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
    data: List[Tuple[str, str]] = []
    all_user_records: Dict[str, Any] = {}
    try:
        admins = Realm.objects.get(string_id=realm_str).get_human_admin_users()
    except Realm.DoesNotExist:
        return HttpResponseNotFound()
    admin_emails = {admin.delivery_email for admin in admins}
    for is_bot, page_title in [(False, "Humans"), (True, "Bots")]:
        all_records = get_user_activity_records_for_realm(realm_str, is_bot)
        user_records, content = realm_user_summary_table(all_records, admin_emails)
        all_user_records.update(user_records)
        data += [(page_title, content)]
    page_title = "Clients"
    content = realm_client_table(all_user_records)
    data += [(page_title, content)]
    page_title = "History"
    content = sent_messages_report(realm_str)
    data += [(page_title, content)]
    title = realm_str
    realm_stats = realm_stats_link(realm_str)
    return render(
        request,
        "analytics/activity.html",
        context=dict(data=data, realm_stats_link=realm_stats, title=title),
    )
--- a/analytics/views/stats.py
+++ b/analytics/views/stats.py
@@ -1,542 +0,0 @@
 import logging
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
 from django.conf import settings
 from django.db.models import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils import translation
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
 from analytics.models import (
    BaseCount,
    InstallationCount,
    RealmCount,
    StreamCount,
    UserCount,
    installation_epoch,
 )
 from zerver.decorator import (
    require_non_guest_user,
    require_server_admin,
    require_server_admin_api,
    to_utc_datetime,
    zulip_login_required,
 )
 from zerver.lib.exceptions import JsonableError
 from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
 from zerver.lib.timestamp import convert_to_UTC
 from zerver.lib.validator import to_non_negative_int
 from zerver.models import Client, Realm, UserProfile, get_realm
 if settings.ZILENCER_ENABLED:
    from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
 MAX_TIME_FOR_FULL_ANALYTICS_GENERATION = timedelta(days=1, minutes=30)
 def is_analytics_ready(realm: Realm) -> bool:
    return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
 def render_stats(
    request: HttpRequest,
    data_url_suffix: str,
    realm: Optional[Realm],
    *,
    title: Optional[str] = None,
    for_installation: bool = False,
    remote: bool = False,
    analytics_ready: bool = True,
 ) -> HttpResponse:
    assert request.user.is_authenticated
    if realm is not None:
        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
        guest_users = UserProfile.objects.filter(
            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
        ).count()
        space_used = realm.currently_used_upload_space_bytes()
        if title:
            pass
        else:
            title = realm.name or realm.string_id
    else:
        assert title
        guest_users = None
        space_used = None
    page_params = dict(
        data_url_suffix=data_url_suffix,
        for_installation=for_installation,
        remote=remote,
        upload_space_used=space_used,
        guest_users=guest_users,
    )
    request_language = get_and_set_request_language(
        request,
        request.user.default_language,
        translation.get_language_from_path(request.path_info),
    )
    page_params["translation_data"] = get_language_translation_data(request_language)
    return render(
        request,
        "analytics/stats.html",
        context=dict(
            target_name=title,
            page_params=page_params,
            analytics_ready=analytics_ready,
        ),
    )
@zulip_login_required
 def stats(request: HttpRequest) -> HttpResponse:
    assert request.user.is_authenticated
    realm = request.user.realm
    if request.user.is_guest:
        # TODO: Make @zulip_login_required pass the UserProfile so we
        # can use @require_member_or_admin
        raise JsonableError(_("Not allowed for guest users"))
    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
@require_server_admin
@has_request_variables
 def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
    try:
        realm = get_realm(realm_str)
    except Realm.DoesNotExist:
        return HttpResponseNotFound()
    return render_stats(
        request,
        f"/realm/{realm_str}",
        realm,
        analytics_ready=is_analytics_ready(realm),
    )
@require_server_admin
@has_request_variables
 def stats_for_remote_realm(
    request: HttpRequest, remote_server_id: int, remote_realm_id: int
 ) -> HttpResponse:
    assert settings.ZILENCER_ENABLED
    server = RemoteZulipServer.objects.get(id=remote_server_id)
    return render_stats(
        request,
        f"/remote/{server.id}/realm/{remote_realm_id}",
        None,
        title=f"Realm {remote_realm_id} on server {server.hostname}",
    )
@require_server_admin_api
@has_request_variables
 def get_chart_data_for_realm(
    request: HttpRequest, /, user_profile: UserProfile, realm_str: str, **kwargs: Any
 ) -> HttpResponse:
    try:
        realm = get_realm(realm_str)
    except Realm.DoesNotExist:
        raise JsonableError(_("Invalid organization"))
    return get_chart_data(request, user_profile, realm=realm, **kwargs)
@require_server_admin_api
@has_request_variables
 def get_chart_data_for_remote_realm(
    request: HttpRequest,
    /,
    user_profile: UserProfile,
    remote_server_id: int,
    remote_realm_id: int,
    **kwargs: Any,
 ) -> HttpResponse:
    assert settings.ZILENCER_ENABLED
    server = RemoteZulipServer.objects.get(id=remote_server_id)
    return get_chart_data(
        request,
        user_profile,
        server=server,
        remote=True,
        remote_realm_id=int(remote_realm_id),
        **kwargs,
    )
@require_server_admin
 def stats_for_installation(request: HttpRequest) -> HttpResponse:
    assert request.user.is_authenticated
    return render_stats(request, "/installation", None, title="installation", for_installation=True)
@require_server_admin
 def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -> HttpResponse:
    assert settings.ZILENCER_ENABLED
    server = RemoteZulipServer.objects.get(id=remote_server_id)
    return render_stats(
        request,
        f"/remote/{server.id}/installation",
        None,
        title=f"remote installation {server.hostname}",
        for_installation=True,
        remote=True,
    )
@require_server_admin_api
@has_request_variables
 def get_chart_data_for_installation(
    request: HttpRequest, /, user_profile: UserProfile, chart_name: str = REQ(), **kwargs: Any
 ) -> HttpResponse:
    return get_chart_data(request, user_profile, for_installation=True, **kwargs)
@require_server_admin_api
@has_request_variables
 def get_chart_data_for_remote_installation(
    request: HttpRequest,
    /,
    user_profile: UserProfile,
    remote_server_id: int,
    chart_name: str = REQ(),
    **kwargs: Any,
 ) -> HttpResponse:
    assert settings.ZILENCER_ENABLED
    server = RemoteZulipServer.objects.get(id=remote_server_id)
    return get_chart_data(
        request,
        user_profile,
        for_installation=True,
        remote=True,
        server=server,
        **kwargs,
    )
@require_non_guest_user
@has_request_variables
 def get_chart_data(
    request: HttpRequest,
    user_profile: UserProfile,
    chart_name: str = REQ(),
    min_length: Optional[int] = REQ(converter=to_non_negative_int, default=None),
    start: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
    end: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
    realm: Optional[Realm] = None,
    for_installation: bool = False,
    remote: bool = False,
    remote_realm_id: Optional[int] = None,
    server: Optional["RemoteZulipServer"] = None,
 ) -> HttpResponse:
    TableType = Union[
        Type["RemoteInstallationCount"],
        Type[InstallationCount],
        Type["RemoteRealmCount"],
        Type[RealmCount],
    ]
    if for_installation:
        if remote:
            assert settings.ZILENCER_ENABLED
            aggregate_table: TableType = RemoteInstallationCount
            assert server is not None
        else:
            aggregate_table = InstallationCount
    else:
        if remote:
            assert settings.ZILENCER_ENABLED
            aggregate_table = RemoteRealmCount
            assert server is not None
            assert remote_realm_id is not None
        else:
            aggregate_table = RealmCount
    tables: Union[Tuple[TableType], Tuple[TableType, Type[UserCount]]]
    if chart_name == "number_of_humans":
        stats = [
            COUNT_STATS["1day_actives::day"],
            COUNT_STATS["realm_active_humans::day"],
            COUNT_STATS["active_users_audit:is_bot:day"],
        ]
        tables = (aggregate_table,)
        subgroup_to_label: Dict[CountStat, Dict[Optional[str], str]] = {
            stats[0]: {None: "_1day"},
            stats[1]: {None: "_15day"},
            stats[2]: {"false": "all_time"},
        }
        labels_sort_function = None
        include_empty_subgroups = True
    elif chart_name == "messages_sent_over_time":
        stats = [COUNT_STATS["messages_sent:is_bot:hour"]]
        tables = (aggregate_table, UserCount)
        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
        labels_sort_function = None
        include_empty_subgroups = True
    elif chart_name == "messages_sent_by_message_type":
        stats = [COUNT_STATS["messages_sent:message_type:day"]]
        tables = (aggregate_table, UserCount)
        subgroup_to_label = {
            stats[0]: {
                "public_stream": _("Public streams"),
                "private_stream": _("Private streams"),
                "private_message": _("Direct messages"),
                "huddle_message": _("Group direct messages"),
            }
        }
        labels_sort_function = lambda data: sort_by_totals(data["everyone"])
        include_empty_subgroups = True
    elif chart_name == "messages_sent_by_client":
        stats = [COUNT_STATS["messages_sent:client:day"]]
        tables = (aggregate_table, UserCount)
        # Note that the labels are further re-written by client_label_map
        subgroup_to_label = {
            stats[0]: {str(id): name for id, name in Client.objects.values_list("id", "name")}
        }
        labels_sort_function = sort_client_labels
        include_empty_subgroups = False
    elif chart_name == "messages_read_over_time":
        stats = [COUNT_STATS["messages_read::hour"]]
        tables = (aggregate_table, UserCount)
        subgroup_to_label = {stats[0]: {None: "read"}}
        labels_sort_function = None
        include_empty_subgroups = True
    else:
        raise JsonableError(_("Unknown chart name: {}").format(chart_name))
    # Most likely someone using our API endpoint. The /stats page does not
    # pass a start or end in its requests.
    if start is not None:
        start = convert_to_UTC(start)
    if end is not None:
        end = convert_to_UTC(end)
    if start is not None and end is not None and start > end:
        raise JsonableError(
            _("Start time is later than end time. Start: {start}, End: {end}").format(
                start=start,
                end=end,
            )
        )
    if realm is None:
        # Note that this value is invalid for Remote tables; be
        # careful not to access it in those code paths.
        realm = user_profile.realm
    if remote:
        # For remote servers, we don't have fillstate data, and thus
        # should simply use the first and last data points for the
        # table.
        assert server is not None
        assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
        aggregate_table_remote = cast(
            Union[Type[RemoteInstallationCount], Type[RemoteRealmCount]], aggregate_table
        )  # https://stackoverflow.com/questions/68540528/mypy-assertions-on-the-types-of-types
        if not aggregate_table_remote.objects.filter(server=server).exists():
            raise JsonableError(
                _("No analytics data available. Please contact your server administrator.")
            )
        if start is None:
            first = aggregate_table_remote.objects.filter(server=server).first()
            assert first is not None
            start = first.end_time
        if end is None:
            last = aggregate_table_remote.objects.filter(server=server).last()
            assert last is not None
            end = last.end_time
    else:
        # Otherwise, we can use tables on the current server to
        # determine a nice range, and some additional validation.
        if start is None:
            if for_installation:
                start = installation_epoch()
            else:
                start = realm.date_created
        if end is None:
            end = max(
                stat.last_successful_fill() or datetime.min.replace(tzinfo=timezone.utc)
                for stat in stats
            )
        if start > end and (timezone_now() - start > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION):
            logging.warning(
                "User from realm %s attempted to access /stats, but the computed "
                "start time: %s (creation of realm or installation) is later than the computed "
                "end time: %s (last successful analytics update). Is the "
                "analytics cron job running?",
                realm.string_id,
                start,
                end,
            )
            raise JsonableError(
                _("No analytics data available. Please contact your server administrator.")
            )
    assert len({stat.frequency for stat in stats}) == 1
    end_times = time_range(start, end, stats[0].frequency, min_length)
    data: Dict[str, Any] = {
        "end_times": [int(end_time.timestamp()) for end_time in end_times],
        "frequency": stats[0].frequency,
    }
    aggregation_level = {
        InstallationCount: "everyone",
        RealmCount: "everyone",
        UserCount: "user",
    }
    if settings.ZILENCER_ENABLED:
        aggregation_level[RemoteInstallationCount] = "everyone"
        aggregation_level[RemoteRealmCount] = "everyone"
    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
    id_value = {
        InstallationCount: -1,
        RealmCount: realm.id,
        UserCount: user_profile.id,
    }
    if settings.ZILENCER_ENABLED:
        if server is not None:
            id_value[RemoteInstallationCount] = server.id
        # TODO: RemoteRealmCount logic doesn't correctly handle
        # filtering by server_id as well.
        if remote_realm_id is not None:
            id_value[RemoteRealmCount] = remote_realm_id
    for table in tables:
        data[aggregation_level[table]] = {}
        for stat in stats:
            data[aggregation_level[table]].update(
                get_time_series_by_subgroup(
                    stat,
                    table,
                    id_value[table],
                    end_times,
                    subgroup_to_label[stat],
                    include_empty_subgroups,
                )
            )
    if labels_sort_function is not None:
        data["display_order"] = labels_sort_function(data)
    else:
        data["display_order"] = None
    return json_success(request, data=data)
 def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
    totals = [(sum(values), label) for label, values in value_arrays.items()]
    totals.sort(reverse=True)
    return [label for total, label in totals]
 # For any given user, we want to show a fixed set of clients in the chart,
 # regardless of the time aggregation or whether we're looking at realm or
 # user data. This fixed set ideally includes the clients most important in
 # understanding the realm's traffic and the user's traffic. This function
 # tries to rank the clients so that taking the first N elements of the
 # sorted list has a reasonable chance of doing so.
 def sort_client_labels(data: Dict[str, Dict[str, List[int]]]) -> List[str]:
    realm_order = sort_by_totals(data["everyone"])
    user_order = sort_by_totals(data["user"])
    label_sort_values: Dict[str, float] = {}
    for i, label in enumerate(realm_order):
        label_sort_values[label] = i
    for i, label in enumerate(user_order):
        label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
    return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
 CountT = TypeVar("CountT", bound=BaseCount)
 def table_filtered_to_id(table: Type[CountT], key_id: int) -> QuerySet[CountT]:
    if table == RealmCount:
        return table.objects.filter(realm_id=key_id)
    elif table == UserCount:
        return table.objects.filter(user_id=key_id)
    elif table == StreamCount:
        return table.objects.filter(stream_id=key_id)
    elif table == InstallationCount:
        return table.objects.all()
    elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
        return table.objects.filter(server_id=key_id)
    elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
        return table.objects.filter(realm_id=key_id)
    else:
        raise AssertionError(f"Unknown table: {table}")
 def client_label_map(name: str) -> str:
    if name == "website":
        return "Web app"
    if name.startswith("desktop app"):
        return "Old desktop app"
    if name == "ZulipElectron":
        return "Desktop app"
    if name == "ZulipTerminal":
        return "Terminal app"
    if name == "ZulipAndroid":
        return "Old Android app"
    if name == "ZulipiOS":
        return "Old iOS app"
    if name == "ZulipMobile":
        return "Mobile app"
    if name in ["ZulipPython", "API: Python"]:
        return "Python API"
    if name.startswith("Zulip") and name.endswith("Webhook"):
        return name[len("Zulip") : -len("Webhook")] + " webhook"
    return name
 def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[int]]:
    mapped_arrays: Dict[str, List[int]] = {}
    for label, array in value_arrays.items():
        mapped_label = client_label_map(label)
        if mapped_label in mapped_arrays:
            for i in range(len(array)):
                mapped_arrays[mapped_label][i] += value_arrays[label][i]
        else:
            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
    return mapped_arrays
 def get_time_series_by_subgroup(
    stat: CountStat,
    table: Type[BaseCount],
    key_id: int,
    end_times: List[datetime],
    subgroup_to_label: Dict[Optional[str], str],
    include_empty_subgroups: bool,
 ) -> Dict[str, List[int]]:
    queryset = (
        table_filtered_to_id(table, key_id)
        .filter(property=stat.property)
        .values_list("subgroup", "end_time", "value")
    )
    value_dicts: Dict[Optional[str], Dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
    for subgroup, end_time, value in queryset:
        value_dicts[subgroup][end_time] = value
    value_arrays = {}
    for subgroup, label in subgroup_to_label.items():
        if (subgroup in value_dicts) or include_empty_subgroups:
            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
    if stat == COUNT_STATS["messages_sent:client:day"]:
        # HACK: We rewrite these arrays to collapse the Client objects
        # with similar names into a single sum, and generally give
        # them better names
        return rewrite_client_arrays(value_arrays)
    return value_arrays
--- a/analytics/views/support.py
+++ b/analytics/views/support.py
@@ -1,408 +0,0 @@
 import urllib
 from contextlib import suppress
 from dataclasses import dataclass
 from datetime import timedelta
 from decimal import Decimal
 from typing import Any, Dict, Iterable, List, Optional
 from urllib.parse import urlencode
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import URLValidator
 from django.db.models import Q
 from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import reverse
 from django.utils.timesince import timesince
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
 from confirmation.models import Confirmation, confirmation_url
 from confirmation.settings import STATUS_USED
 from zerver.actions.create_realm import do_change_realm_subdomain
 from zerver.actions.realm_settings import (
    do_change_realm_org_type,
    do_change_realm_plan_type,
    do_deactivate_realm,
    do_scrub_realm,
    do_send_realm_reactivation_email,
 )
 from zerver.actions.users import do_delete_user_preserving_messages
 from zerver.decorator import require_server_admin
 from zerver.forms import check_subdomain_available
 from zerver.lib.exceptions import JsonableError
 from zerver.lib.realm_icon import realm_icon_url
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.subdomains import get_subdomain_from_hostname
 from zerver.lib.validator import check_bool, check_string_in, to_decimal, to_non_negative_int
 from zerver.models import (
    MultiuseInvite,
    PreregistrationRealm,
    PreregistrationUser,
    Realm,
    RealmReactivationStatus,
    UserProfile,
    get_org_type_display_name,
    get_realm,
    get_user_profile_by_id,
 )
 from zerver.views.invite import get_invitee_emails_set
 if settings.BILLING_ENABLED:
    from corporate.lib.stripe import approve_sponsorship as do_approve_sponsorship
    from corporate.lib.stripe import (
        attach_discount_to_realm,
        downgrade_at_the_end_of_billing_cycle,
        downgrade_now_without_creating_additional_invoices,
        get_discount_for_realm,
        get_latest_seat_count,
        make_end_of_cycle_updates_if_needed,
        switch_realm_from_standard_to_plus_plan,
        update_billing_method_of_current_plan,
        update_sponsorship_status,
        void_all_open_invoices,
    )
    from corporate.models import (
        Customer,
        CustomerPlan,
        get_current_plan_by_realm,
        get_customer_by_realm,
    )
 def get_plan_name(plan_type: int) -> str:
    return {
        Realm.PLAN_TYPE_SELF_HOSTED: "self-hosted",
        Realm.PLAN_TYPE_LIMITED: "limited",
        Realm.PLAN_TYPE_STANDARD: "standard",
        Realm.PLAN_TYPE_STANDARD_FREE: "open source",
        Realm.PLAN_TYPE_PLUS: "plus",
    }[plan_type]
 def get_confirmations(
    types: List[int], object_ids: Iterable[int], hostname: Optional[str] = None
 ) -> List[Dict[str, Any]]:
    lowest_datetime = timezone_now() - timedelta(days=30)
    confirmations = Confirmation.objects.filter(
        type__in=types, object_id__in=object_ids, date_sent__gte=lowest_datetime
    )
    confirmation_dicts = []
    for confirmation in confirmations:
        realm = confirmation.realm
        content_object = confirmation.content_object
        type = confirmation.type
        expiry_date = confirmation.expiry_date
        assert content_object is not None
        if hasattr(content_object, "status"):
            if content_object.status == STATUS_USED:
                link_status = "Link has been used"
            else:
                link_status = "Link has not been used"
        else:
            link_status = ""
        now = timezone_now()
        if expiry_date is None:
            expires_in = "Never"
        elif now < expiry_date:
            expires_in = timesince(now, expiry_date)
        else:
            expires_in = "Expired"
        url = confirmation_url(confirmation.confirmation_key, realm, type)
        confirmation_dicts.append(
            {
                "object": confirmation.content_object,
                "url": url,
                "type": type,
                "link_status": link_status,
                "expires_in": expires_in,
            }
        )
    return confirmation_dicts
 VALID_MODIFY_PLAN_METHODS = [
    "downgrade_at_billing_cycle_end",
    "downgrade_now_without_additional_licenses",
    "downgrade_now_void_open_invoices",
    "upgrade_to_plus",
 ]
 VALID_STATUS_VALUES = [
    "active",
    "deactivated",
 ]
 VALID_BILLING_METHODS = [
    "send_invoice",
    "charge_automatically",
 ]
@dataclass
 class PlanData:
    customer: Optional["Customer"] = None
    current_plan: Optional["CustomerPlan"] = None
    licenses: Optional[int] = None
    licenses_used: Optional[int] = None
@require_server_admin
@has_request_variables
 def support(
    request: HttpRequest,
    realm_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
    plan_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
    discount: Optional[Decimal] = REQ(default=None, converter=to_decimal),
    new_subdomain: Optional[str] = REQ(default=None),
    status: Optional[str] = REQ(default=None, str_validator=check_string_in(VALID_STATUS_VALUES)),
    billing_method: Optional[str] = REQ(
        default=None, str_validator=check_string_in(VALID_BILLING_METHODS)
    ),
    sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
    approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
    modify_plan: Optional[str] = REQ(
        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
    ),
    scrub_realm: bool = REQ(default=False, json_validator=check_bool),
    delete_user_by_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
    query: Optional[str] = REQ("q", default=None),
    org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
 ) -> HttpResponse:
    context: Dict[str, Any] = {}
    if "success_message" in request.session:
        context["success_message"] = request.session["success_message"]
        del request.session["success_message"]
    if settings.BILLING_ENABLED and request.method == "POST":
        # We check that request.POST only has two keys in it: The
        # realm_id and a field to change.
        keys = set(request.POST.keys())
        if "csrfmiddlewaretoken" in keys:
            keys.remove("csrfmiddlewaretoken")
        if len(keys) != 2:
            raise JsonableError(_("Invalid parameters"))
        assert realm_id is not None
        realm = Realm.objects.get(id=realm_id)
        acting_user = request.user
        assert isinstance(acting_user, UserProfile)
        if plan_type is not None:
            current_plan_type = realm.plan_type
            do_change_realm_plan_type(realm, plan_type, acting_user=acting_user)
            msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(plan_type)} "
            context["success_message"] = msg
        elif org_type is not None:
            current_realm_type = realm.org_type
            do_change_realm_org_type(realm, org_type, acting_user=acting_user)
            msg = f"Org type of {realm.string_id} changed from {get_org_type_display_name(current_realm_type)} to {get_org_type_display_name(org_type)} "
            context["success_message"] = msg
        elif discount is not None:
            current_discount = get_discount_for_realm(realm) or 0
            attach_discount_to_realm(realm, discount, acting_user=acting_user)
            context[
                "success_message"
            ] = f"Discount of {realm.string_id} changed to {discount}% from {current_discount}%."
        elif new_subdomain is not None:
            old_subdomain = realm.string_id
            try:
                check_subdomain_available(new_subdomain)
            except ValidationError as error:
                context["error_message"] = error.message
            else:
                do_change_realm_subdomain(realm, new_subdomain, acting_user=acting_user)
                request.session[
                    "success_message"
                ] = f"Subdomain changed from {old_subdomain} to {new_subdomain}"
                return HttpResponseRedirect(
                    reverse("support") + "?" + urlencode({"q": new_subdomain})
                )
        elif status is not None:
            if status == "active":
                do_send_realm_reactivation_email(realm, acting_user=acting_user)
                context[
                    "success_message"
                ] = f"Realm reactivation email sent to admins of {realm.string_id}."
            elif status == "deactivated":
                do_deactivate_realm(realm, acting_user=acting_user)
                context["success_message"] = f"{realm.string_id} deactivated."
        elif billing_method is not None:
            if billing_method == "send_invoice":
                update_billing_method_of_current_plan(
                    realm, charge_automatically=False, acting_user=acting_user
                )
                context[
                    "success_message"
                ] = f"Billing method of {realm.string_id} updated to pay by invoice."
            elif billing_method == "charge_automatically":
                update_billing_method_of_current_plan(
                    realm, charge_automatically=True, acting_user=acting_user
                )
                context[
                    "success_message"
                ] = f"Billing method of {realm.string_id} updated to charge automatically."
        elif sponsorship_pending is not None:
            if sponsorship_pending:
                update_sponsorship_status(realm, True, acting_user=acting_user)
                context["success_message"] = f"{realm.string_id} marked as pending sponsorship."
            else:
                update_sponsorship_status(realm, False, acting_user=acting_user)
                context["success_message"] = f"{realm.string_id} is no longer pending sponsorship."
        elif approve_sponsorship:
            do_approve_sponsorship(realm, acting_user=acting_user)
            context["success_message"] = f"Sponsorship approved for {realm.string_id}"
        elif modify_plan is not None:
            if modify_plan == "downgrade_at_billing_cycle_end":
                downgrade_at_the_end_of_billing_cycle(realm)
                context[
                    "success_message"
                ] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
            elif modify_plan == "downgrade_now_without_additional_licenses":
                downgrade_now_without_creating_additional_invoices(realm)
                context[
                    "success_message"
                ] = f"{realm.string_id} downgraded without creating additional invoices"
            elif modify_plan == "downgrade_now_void_open_invoices":
                downgrade_now_without_creating_additional_invoices(realm)
                voided_invoices_count = void_all_open_invoices(realm)
                context[
                    "success_message"
                ] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
            elif modify_plan == "upgrade_to_plus":
                switch_realm_from_standard_to_plus_plan(realm)
                context["success_message"] = f"{realm.string_id} upgraded to Plus"
        elif scrub_realm:
            do_scrub_realm(realm, acting_user=acting_user)
            context["success_message"] = f"{realm.string_id} scrubbed."
        elif delete_user_by_id:
            user_profile_for_deletion = get_user_profile_by_id(delete_user_by_id)
            user_email = user_profile_for_deletion.delivery_email
            assert user_profile_for_deletion.realm == realm
            do_delete_user_preserving_messages(user_profile_for_deletion)
            context["success_message"] = f"{user_email} in {realm.subdomain} deleted."
    if query:
        key_words = get_invitee_emails_set(query)
        case_insensitive_users_q = Q()
        for key_word in key_words:
            case_insensitive_users_q |= Q(delivery_email__iexact=key_word)
        users = set(UserProfile.objects.filter(case_insensitive_users_q))
        realms = set(Realm.objects.filter(string_id__in=key_words))
        for key_word in key_words:
            try:
                URLValidator()(key_word)
                parse_result = urllib.parse.urlparse(key_word)
                hostname = parse_result.hostname
                assert hostname is not None
                if parse_result.port:
                    hostname = f"{hostname}:{parse_result.port}"
                subdomain = get_subdomain_from_hostname(hostname)
                with suppress(Realm.DoesNotExist):
                    realms.add(get_realm(subdomain))
            except ValidationError:
                users.update(UserProfile.objects.filter(full_name__iexact=key_word))
        # full_names can have , in them
        users.update(UserProfile.objects.filter(full_name__iexact=query))
        context["users"] = users
        context["realms"] = realms
        confirmations: List[Dict[str, Any]] = []
        preregistration_user_ids = [
            user.id for user in PreregistrationUser.objects.filter(email__in=key_words)
        ]
        confirmations += get_confirmations(
            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION],
            preregistration_user_ids,
            hostname=request.get_host(),
        )
        preregistration_realm_ids = [
            user.id for user in PreregistrationRealm.objects.filter(email__in=key_words)
        ]
        confirmations += get_confirmations(
            [Confirmation.REALM_CREATION],
            preregistration_realm_ids,
            hostname=request.get_host(),
        )
        multiuse_invite_ids = [
            invite.id for invite in MultiuseInvite.objects.filter(realm__in=realms)
        ]
        confirmations += get_confirmations([Confirmation.MULTIUSE_INVITE], multiuse_invite_ids)
        realm_reactivation_status_objects = RealmReactivationStatus.objects.filter(realm__in=realms)
        confirmations += get_confirmations(
            [Confirmation.REALM_REACTIVATION], [obj.id for obj in realm_reactivation_status_objects]
        )
        context["confirmations"] = confirmations
        # We want a union of all realms that might appear in the search result,
        # but not necessary as a separate result item.
        # Therefore, we do not modify the realms object in the context.
        all_realms = realms.union(
            [
                confirmation["object"].realm
                for confirmation in confirmations
                # For confirmations, we only display realm details when the type is USER_REGISTRATION
                # or INVITATION.
                if confirmation["type"] in (Confirmation.USER_REGISTRATION, Confirmation.INVITATION)
            ]
            + [user.realm for user in users]
        )
        plan_data: Dict[int, PlanData] = {}
        for realm in all_realms:
            current_plan = get_current_plan_by_realm(realm)
            plan_data[realm.id] = PlanData(
                customer=get_customer_by_realm(realm),
                current_plan=current_plan,
            )
            if current_plan is not None:
                new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(
                    current_plan, timezone_now()
                )
                if last_ledger_entry is not None:
                    if new_plan is not None:
                        plan_data[realm.id].current_plan = new_plan
                    else:
                        plan_data[realm.id].current_plan = current_plan
                    plan_data[realm.id].licenses = last_ledger_entry.licenses
                    plan_data[realm.id].licenses_used = get_latest_seat_count(realm)
        context["plan_data"] = plan_data
    def get_realm_owner_emails_as_string(realm: Realm) -> str:
        return ", ".join(
            realm.get_human_owner_users()
            .order_by("delivery_email")
            .values_list("delivery_email", flat=True)
        )
    def get_realm_admin_emails_as_string(realm: Realm) -> str:
        return ", ".join(
            realm.get_human_admin_users(include_realm_owners=False)
            .order_by("delivery_email")
            .values_list("delivery_email", flat=True)
        )
    context["get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string
    context["get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string
    context["get_discount_for_realm"] = get_discount_for_realm
    context["get_org_type_display_name"] = get_org_type_display_name
    context["realm_icon_url"] = realm_icon_url
    context["Confirmation"] = Confirmation
    context["sorted_realm_types"] = sorted(
        Realm.ORG_TYPES.values(), key=lambda d: d["display_order"]
    )
    return render(request, "analytics/support.html", context=context)
--- a/analytics/views/user_activity.py
+++ b/analytics/views/user_activity.py
@@ -1,106 +0,0 @@
 from typing import Any, Dict, List, Tuple
 from django.conf import settings
 from django.db.models import QuerySet
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 from analytics.views.activity_common import (
    format_date_for_activity_reports,
    get_user_activity_summary,
    make_table,
 )
 from zerver.decorator import require_server_admin
 from zerver.models import UserActivity, UserProfile, get_user_profile_by_id
 if settings.BILLING_ENABLED:
    pass
 def get_user_activity_records(
    user_profile: UserProfile,
 ) -> QuerySet[UserActivity]:
    fields = [
        "user_profile__full_name",
        "query",
        "client__name",
        "count",
        "last_visit",
    ]
    records = UserActivity.objects.filter(
        user_profile=user_profile,
    )
    records = records.order_by("-last_visit")
    records = records.select_related("user_profile", "client").only(*fields)
    return records
 def raw_user_activity_table(records: QuerySet[UserActivity]) -> str:
    cols = [
        "query",
        "client",
        "count",
        "last_visit",
    ]
    def row(record: UserActivity) -> List[Any]:
        return [
            record.query,
            record.client.name,
            record.count,
            format_date_for_activity_reports(record.last_visit),
        ]
    rows = list(map(row, records))
    title = "Raw data"
    return make_table(title, cols, rows)
 def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
    rows = []
    for k, v in user_summary.items():
        if k in ("name", "user_profile_id"):
            continue
        client = k
        count = v["count"]
        last_visit = v["last_visit"]
        row = [
            format_date_for_activity_reports(last_visit),
            client,
            count,
        ]
        rows.append(row)
    rows = sorted(rows, key=lambda r: r[0], reverse=True)
    cols = [
        "last_visit",
        "client",
        "count",
    ]
    title = "User activity"
    return make_table(title, cols, rows)
@require_server_admin
 def get_user_activity(request: HttpRequest, user_profile_id: int) -> HttpResponse:
    user_profile = get_user_profile_by_id(user_profile_id)
    records = get_user_activity_records(user_profile)
    data: List[Tuple[str, str]] = []
    user_summary = get_user_activity_summary(records)
    content = user_activity_summary_table(user_summary)
    data += [("Summary", content)]
    content = raw_user_activity_table(records)
    data += [("Info", content)]
    title = user_profile.delivery_email
    return render(
        request,
        "analytics/activity.html",
        context=dict(data=data, title=title),
    )
--- a/api/MANIFEST.in
+++ b/api/MANIFEST.in
@@ -0,0 +1,11 @@
 recursive-include integrations *
 include README.md
 include examples/zuliprc
 include examples/send-message
 include examples/subscribe
 include examples/get-public-streams
 include examples/unsubscribe
 include examples/list-members
 include examples/list-subscriptions
 include examples/print-messages
 include examples/recent-messages
--- a/api/README.md
+++ b/api/README.md
@@ -0,0 +1,159 @@
 #### Dependencies
 The [Zulip API](https://zulip.com/api) Python bindings require the
 following Python libraries:
 * simplejson
 * requests (version >= 0.12.1)
 #### Installing
 This package uses distutils, so you can just run:
    python setup.py install
 #### Using the API
 For now, the only fully supported API operation is sending a message.
 The other API queries work, but are under active development, so
 please make sure we know you're using them so that we can notify you
 as we make any changes to them.
 The easiest way to use these API bindings is to base your tools off
 of the example tools under examples/ in this distribution.
 If you place your API key in the config file `~/.zuliprc` the Python
 API bindings will automatically read it in. The format of the config
 file is as follows:
    [api]
    key=<api key from the web interface>
    email=<your email address>
    site=<your Zulip server's URI>
    insecure=<true or false, true means do not verify the server certificate>
    cert_bundle=<path to a file containing CA or server certificates to trust>
 If omitted, these settings have the following defaults:
    site=https://api.zulip.com
    insecure=false
    cert_bundle=<the default CA bundle trusted by Python>
 Alternatively, you may explicitly use "--user" and "--api-key" in our
 examples, which is especially useful if you are running several bots
 which share a home directory.
 The command line equivalents for other configuration options are:
    --site=<your Zulip server's URI>
    --insecure
    --cert-bundle=<file>
 You can obtain your Zulip API key, create bots, and manage bots all
 from your Zulip [settings page](https://zulip.com/#settings).
 A typical simple bot sending API messages will look as follows:
 At the top of the file:
    # Make sure the Zulip API distribution's root directory is in sys.path, then:
    import zulip
    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
 When you want to send a message:
    message = {
      "type": "stream",
      "to": ["support"],
      "subject": "your subject",
      "content": "your content",
    }
    zulip_client.send_message(message)
 Additional examples:
    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
                         'subject': 'feedback', 'to': ['support']})
    client.send_message({'type': 'private', 'content': 'Zulip rules!',
                         'to': ['user1@example.com', 'user2@example.com']})
 send_message() returns a dict guaranteed to contain the following
 keys: msg, result.  For successful calls, result will be "success" and
 msg will be the empty string.  On error, result will be "error" and
 msg will describe what went wrong.
 #### Logging
 The Zulip API comes with a ZulipStream class which can be used with the
 logging module:
 ```
 import zulip
 import logging
 stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
 logger = logging.getLogger("your_logger")
 logger.addHandler(logging.StreamHandler(stream))
 logger.setLevel(logging.DEBUG)
 logger.info("This is an INFO test.")
 logger.debug("This is a DEBUG test.")
 logger.warn("This is a WARN test.")
 logger.error("This is a ERROR test.")
 ```
 #### Sending messages
 You can use the included `zulip-send` script to send messages via the
 API directly from existing scripts.
    zulip-send hamlet@example.com cordelia@example.com -m \
        "Conscience doth make cowards of us all."
 Alternatively, if you don't want to use your ~/.zuliprc file:
    zulip-send --user shakespeare-bot@example.com \
        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
        hamlet@example.com cordelia@example.com -m \
        "Conscience doth make cowards of us all."
 #### Working with an untrusted server certificate
 If your server has either a self-signed certificate, or a certificate signed
 by a CA that you don't wish to globally trust then by default the API will
 fail with an SSL verification error.
 You can add `insecure=true` to your .zuliprc file.
    [api]
    site=https://zulip.example.com
    insecure=true
 This disables verification of the server certificate, so connections are
 encrypted but unauthenticated. This is not secure, but may be good enough
 for a development environment.
 You can explicitly trust the server certificate using `cert_bundle=<filename>`
 in your .zuliprc file.
    [api]
    site=https://zulip.example.com
    cert_bundle=/home/bots/certs/zulip.example.com.crt
 You can also explicitly trust a different set of Certificate Authorities from
 the default bundle that is trusted by Python. For example to trust a company
 internal CA.
    [api]
    site=https://zulip.example.com
    cert_bundle=/home/bots/certs/example.com.ca-bundle
 Save the server certificate (or the CA certificate) in its own file,
 converting to PEM format first if necessary.
 Verify that the certificate you have saved is the same as the one on the
 server.
 The `cert_bundle` option trusts the server / CA certificate only for
 interaction with the zulip site, and is relatively secure.
 Note that a certificate bundle is merely one or more certificates combined
 into a single file.
--- a/api/bin/zulip-send
+++ b/api/bin/zulip-send
@@ -0,0 +1,126 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # zulip-send -- Sends a message to the specified recipients.
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import sys
 import os
 import optparse
 import logging
 logging.basicConfig()
 log = logging.getLogger('zulip-send')
 def do_send_message(client, message_data ):
    '''Sends a message and optionally prints status about the same.'''
    if message_data['type'] == 'stream':
        log.info('Sending message to stream "%s", subject "%s"... ' % \
            (message_data['to'], message_data['subject']))
    else:
        log.info('Sending message to %s... ' % message_data['to'])
    response = client.send_message(message_data)
    if response['result'] == 'success':
        log.info('Message sent.')
        return True
    else:
        log.error(response['msg'])
        return False
 def main(argv=None):
    if argv is None:
        argv = sys.argv
    usage = """%prog [options] [recipient...]
    Sends a message specified recipients.
    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
    '--user' and '--api-key' arguments.
    """
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
    import zulip
    parser = optparse.OptionParser(usage=usage)
    # Grab parser options from the API common set
    parser.add_option_group(zulip.generate_option_group(parser))
    parser.add_option('-m', '--message',
                      help='Specifies the message to send, prevents interactive prompting.')
    group = optparse.OptionGroup(parser, 'Stream parameters')
    group.add_option('-s', '--stream',
                      dest='stream',
                      action='store',
                      help='Allows the user to specify a stream for the message.')
    group.add_option('-S', '--subject',
                      dest='subject',
                      action='store',
                      help='Allows the user to specify a subject for the message.')
    parser.add_option_group(group)
    (options, recipients) = parser.parse_args(argv[1:])
    if options.verbose:
        logging.getLogger().setLevel(logging.INFO)
    # Sanity check user data
    if len(recipients) != 0 and (options.stream or options.subject):
        parser.error('You cannot specify both a username and a stream/subject.')
    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
        parser.error('Stream messages must have a subject')
    if len(recipients) == 0 and not (options.stream and options.subject):
        parser.error('You must specify a stream/subject or at least one recipient.')
    client = zulip.init_from_options(options)
    if not options.message:
        options.message = sys.stdin.read()
    if options.stream:
        message_data = {
            'type': 'stream',
            'content': options.message,
            'subject': options.subject,
            'to': options.stream,
        }
    else:
        message_data = {
            'type': 'private',
            'content': options.message,
            'to': recipients,
        }
    if not do_send_message(client, message_data):
        return 1
 if __name__ == '__main__':
    sys.exit(main())
--- a/api/examples/create-user
+++ b/api/examples/create-user
@@ -0,0 +1,55 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012-2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 from os import path
 import optparse
 usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
 Create a user. You must be a realm admin to use this API, and the user
 will be created in your realm.
 Example: create-user --site=http://localhost:9991 --user=rwbarton@zulip.com --new-email=jarthur@zulip.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
 """
 sys.path.append(path.join(path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--new-email')
 parser.add_option('--new-password')
 parser.add_option('--new-full-name')
 parser.add_option('--new-short-name')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print(client.create_user({
        'email': options.new_email,
        'password': options.new_password,
        'full_name': options.new_full_name,
        'short_name': options.new_short_name
        }))
--- a/api/examples/edit-message
+++ b/api/examples/edit-message
@@ -0,0 +1,57 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
 Edits a message that you sent
 Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option('--message-id', default="")
 parser.add_option('--subject', default="")
 parser.add_option('--content',   default="")
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 message_data = {
    "message_id": options.message_id,
 }
 if options.subject != "":
    message_data["subject"] = options.subject
 if options.content != "":
    message_data["content"] = options.content
 print(client.update_message(message_data))
--- a/api/examples/get-public-streams
+++ b/api/examples/get-public-streams
@@ -0,0 +1,47 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out all the public streams in the realm.
 Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print(client.get_streams(include_public=True, include_subscribed=False))
--- a/api/examples/list-members
+++ b/api/examples/list-members
@@ -0,0 +1,46 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
 List the names and e-mail addresses of the people in your realm.
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 for user in client.get_members()["members"]:
    print(user["full_name"], user["email"])
--- a/api/examples/list-subscriptions
+++ b/api/examples/list-subscriptions
@@ -0,0 +1,46 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out a list of the user's subscriptions.
 Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print(client.list_subscriptions())
--- a/api/examples/print-events
+++ b/api/examples/print-events
@@ -0,0 +1,52 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out certain events received by the indicated bot or user matching the filter below.
 Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 def print_event(event):
    print(event)
 # This is a blocking call, and will continuously poll for new events
 # Note also the filter here is messages to the stream Denmark; if you
 # don't specify event_types it'll print all events.
 client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])
--- a/api/examples/print-messages
+++ b/api/examples/print-messages
@@ -0,0 +1,50 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out each message received by the indicated bot or user.
 Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 def print_message(message):
    print(message)
 # This is a blocking call, and will continuously poll for new messages
 client.call_on_each_message(print_message)
--- a/api/examples/print-next-message
+++ b/api/examples/print-next-message
@@ -0,0 +1,46 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """print-next-message --user=<bot's email address> --api-key=<bot's api key> [options]
 Prints out the next message received by the user.
 Example: print-next-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 print(client.get_messages({}))
--- a/api/examples/recent-messages
+++ b/api/examples/recent-messages
@@ -0,0 +1,61 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import json
 import optparse
 usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
 Prints out last count messages recieved by the indicated bot or user
 Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option('--count', default=100)
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 req = {
    'narrow': [["stream", "Denmark"]],
    'num_before': options.count,
    'num_after': 0,
    'anchor': 1000000000,
    'apply_markdown': False
 }
 old_messages = client.do_api_query(req, zulip.API_VERSTRING + 'messages', method='GET')
 if 'messages' in old_messages:
    for message in old_messages['messages']:
        print(json.dumps(message, indent=4))
 else:
    print([])
--- a/api/examples/send-message
+++ b/api/examples/send-message
@@ -0,0 +1,58 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
 Sends a test message to the specified recipients.
 Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
 Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 parser = optparse.OptionParser(usage=usage)
 parser.add_option('--subject', default="test")
 parser.add_option('--message', default="test message")
 parser.add_option('--type',   default='private')
 parser.add_option_group(zulip.generate_option_group(parser))
 (options, args) = parser.parse_args()
 if len(args) == 0:
    parser.error("You must specify recipients")
 client = zulip.init_from_options(options)
 message_data = {
    "type": options.type,
    "content": options.message,
    "subject": options.subject,
    "to": args,
 }
 print(client.send_message(message_data))
--- a/api/examples/subscribe
+++ b/api/examples/subscribe
@@ -0,0 +1,53 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
 Ensures the user is subscribed to the listed streams.
 Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--streams', default='')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 if options.streams == "":
    print("Usage:", parser.usage, file=sys.stderr)
    sys.exit(1)
 print(client.add_subscriptions([{"name": stream_name} for stream_name in
                                options.streams.split()]))
--- a/api/examples/unsubscribe
+++ b/api/examples/unsubscribe
@@ -0,0 +1,52 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # Copyright © 2012 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 from __future__ import print_function
 import sys
 import os
 import optparse
 usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
 Ensures the user is not subscribed to the listed streams.
 Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
 You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
 """
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 import zulip
 parser = optparse.OptionParser(usage=usage)
 parser.add_option_group(zulip.generate_option_group(parser))
 parser.add_option('--streams', default='')
 (options, args) = parser.parse_args()
 client = zulip.init_from_options(options)
 if options.streams == "":
    print("Usage:", parser.usage, file=sys.stderr)
    sys.exit(1)
 print(client.remove_subscriptions(options.streams.split()))
--- a/api/examples/zuliprc
+++ b/api/examples/zuliprc
@@ -0,0 +1,4 @@
 ; Save this file as ~/.zuliprc
 [api]
 key=<your bot's api key from the web interface>
 email=<your bot's email address>
--- a/api/integrations/asana/zulip_asana_config.py
+++ b/api/integrations/asana/zulip_asana_config.py
@@ -0,0 +1,57 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
 # Copyright © 2014 Zulip, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in
 # all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 ### REQUIRED CONFIGURATION ###
 # Change these values to your Asana credentials.
 ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
 # Change these values to the credentials for your Asana bot.
 ZULIP_USER = "asana-bot@example.com"
 ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
 # The Zulip stream that will receive Asana task updates.
 ZULIP_STREAM_NAME = "asana"
 ### OPTIONAL CONFIGURATION ###
 # Set to None for logging to stdout when testing, and to a file for
 # logging in production.
 #LOG_FILE = "/var/tmp/zulip_asana.log"
 LOG_FILE = None
 # This file is used to resume this mirror in case the script shuts down.
 # It is required and needs to be writeable.
 RESUME_FILE = "/var/tmp/zulip_asana.state"
 # When initially started, how many hours of messages to include.
 ASANA_INITIAL_HISTORY_HOURS = 1
 # Set this to your Zulip API server URI
 ZULIP_SITE = "https://api.zulip.com"
 # If properly installed, the Zulip API should be in your import
 # path, but if not, set a custom path below
 ZULIP_API_PATH = None
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tim Abbott	8cc7642cdd	Update changelog for Zulip 1.4.1 and 1.4.2 releases.	2016-09-27 20:09:20 -07:00
Tim Abbott	6883c916af	Upgrade Django to 1.8.15 with Zulip patches.	2016-09-27 20:09:20 -07:00
Tim Abbott	978a568c0f	puppet: Fix buggy logrotate configuration.	2016-09-27 20:01:17 -07:00
Steve Howell	f6975f9334	Upgrade: revert change to default LOCAL_UPLOADS_DIR in prod settings. The main purpose of the "var" convention is to make it easy to write stuff inside of our git repo when running a dev instance, and then "var" gets excluded from checkins. For production, that's not as much of a concern. For upgrades we don't want to be changing the directory around and confusing matters, especially with the extra moving part of nginx configs (which have their own issues in terms of being overwritten by accident when admins go to S3).	2016-09-02 12:30:29 -07:00
Christie Koehler	0120ff5612	upgrade: Create prod_settings symlink in step 2 if it doesn't exist. Between releases 1.3.13 and 1.4.0, local_settings.py was renamed to prod_settings.py. The upgrade scripts were adjusted to reflect this name change. But because the first part of the upgrade script is run with the currently installed version's code, the symlink to /etc/zulip/settings.py is created with the old name. This was causing upgrade-zulip-stage-2 to fail. Now upgrade-zulip-stage-2 creates the symlink at zproject/prod_settings.py if it doesn't already exist. Fixes #1731.	2016-09-01 21:17:11 -07:00
		`@@ -0,0 +1,2 @@`
							`service_name: travis-pro`
							`repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG`