Update changelog for Zulip 1.4.3 release.

A bug in Zulip's implementation of the "stream exists" endpoint meant
that any user of a Zulip server could subscribe to an invite-only
stream without needing to be invited by using the "autosubscribe"
argument.

Thanks to Rafid Aslam for discovering this issue.

.codecov.yml

@@ -1,12 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        # Codecov has the tendency to report a lot of false negatives,
-        # so we basically suppress comments completely.
-        threshold: 50%
-        base: auto
-    patch: off

.codespellignore

@@ -1,27 +0,0 @@
-te
-ans
-pullrequest
-ist
-cros
-wit
-nwe
-circularly
-ned
-ba
-ressemble
-ser
-sur
-hel
-fpr
-alls
-nd
-ot
-womens
-vise
-falsy
-ro
-derails
-forin
-uper
-slac
-couldn

.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG

.editorconfig

@@ -1,24 +0,0 @@
-root = true
-
-[*]
-end_of_line = lf
-charset = utf-8
-indent_size = 4
-indent_style = space
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-binary_next_line = true  # for shfmt
-switch_case_indent = true  # for shfmt
-
-[{*.{js,json,ts},check-openapi}]
-max_line_length = 100
-
-[*.{py,pyi}]
-max_line_length = 110
-
-[*.{md,svg,rb,pp,yaml,yml}]
-indent_size = 2
-
-[package.json]
-indent_size = 2

.eslintignore

@@ -1,15 +0,0 @@
-# This is intended for generated files and vendored third-party files.
-# For our source code, instead of adding files here, consider using
-# specific eslint-disable comments in the files themselves.
-
-/docs/_build
-/static/generated
-/static/webpack-bundles
-/var/*
-!/var/puppeteer
-/var/puppeteer/*
-!/var/puppeteer/test_credentials.d.ts
-/web/generated
-/web/third
-/zulip-current-venv
-/zulip-py3-venv

.eslintrc.json

@@ -1,274 +0,0 @@
-{
-    "root": true,
-    "env": {
-        "es2020": true,
-        "node": true
-    },
-    "extends": [
-        "eslint:recommended",
-        "plugin:import/errors",
-        "plugin:import/warnings",
-        "plugin:no-jquery/recommended",
-        "plugin:no-jquery/deprecated",
-        "plugin:unicorn/recommended",
-        "prettier"
-    ],
-    "parser": "@babel/eslint-parser",
-    "parserOptions": {
-        "requireConfigFile": false,
-        "warnOnUnsupportedTypeScriptVersion": false,
-        "sourceType": "unambiguous"
-    },
-    "plugins": ["formatjs", "no-jquery"],
-    "settings": {
-        "formatjs": {
-            "additionalFunctionNames": ["$t", "$t_html"]
-        },
-        "no-jquery": {
-            "collectionReturningPlugins": {
-                "expectOne": "always"
-            },
-            "variablePattern": "^\\$(?!t$|t_html$)."
-        }
-    },
-    "reportUnusedDisableDirectives": true,
-    "rules": {
-        "array-callback-return": "error",
-        "arrow-body-style": "error",
-        "block-scoped-var": "error",
-        "consistent-return": "error",
-        "curly": "error",
-        "dot-notation": "error",
-        "eqeqeq": "error",
-        "formatjs/enforce-default-message": ["error", "literal"],
-        "formatjs/enforce-placeholders": [
-            "error",
-            {"ignoreList": ["b", "code", "em", "i", "kbd", "p", "strong"]}
-        ],
-        "formatjs/no-id": "error",
-        "guard-for-in": "error",
-        "import/extensions": "error",
-        "import/first": "error",
-        "import/newline-after-import": "error",
-        "import/no-self-import": "error",
-        "import/no-unresolved": "off",
-        "import/no-useless-path-segments": "error",
-        "import/order": ["error", {"alphabetize": {"order": "asc"}, "newlines-between": "always"}],
-        "import/unambiguous": "error",
-        "lines-around-directive": "error",
-        "new-cap": "error",
-        "no-alert": "error",
-        "no-array-constructor": "error",
-        "no-bitwise": "error",
-        "no-caller": "error",
-        "no-catch-shadow": "error",
-        "no-constant-condition": ["error", {"checkLoops": false}],
-        "no-div-regex": "error",
-        "no-duplicate-imports": "error",
-        "no-else-return": "error",
-        "no-eq-null": "error",
-        "no-eval": "error",
-        "no-implicit-coercion": "error",
-        "no-implied-eval": "error",
-        "no-inner-declarations": "off",
-        "no-iterator": "error",
-        "no-jquery/no-constructor-attributes": "error",
-        "no-jquery/no-parse-html-literal": "error",
-        "no-label-var": "error",
-        "no-labels": "error",
-        "no-loop-func": "error",
-        "no-multi-str": "error",
-        "no-native-reassign": "error",
-        "no-new-func": "error",
-        "no-new-object": "error",
-        "no-new-wrappers": "error",
-        "no-octal-escape": "error",
-        "no-plusplus": "error",
-        "no-proto": "error",
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-self-compare": "error",
-        "no-sync": "error",
-        "no-throw-literal": "error",
-        "no-undef-init": "error",
-        "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
-        "no-unused-expressions": "error",
-        "no-unused-vars": ["error", {"ignoreRestSiblings": true}],
-        "no-use-before-define": ["error", {"functions": false}],
-        "no-useless-concat": "error",
-        "no-useless-constructor": "error",
-        "no-var": "error",
-        "object-shorthand": ["error", "always", {"avoidExplicitReturnArrows": true}],
-        "one-var": ["error", "never"],
-        "prefer-arrow-callback": "error",
-        "prefer-const": ["error", {"ignoreReadBeforeAssign": true}],
-        "radix": "error",
-        "sort-imports": ["error", {"ignoreDeclarationSort": true}],
-        "spaced-comment": ["error", "always", {"markers": ["/"]}],
-        "strict": "error",
-        "unicorn/consistent-function-scoping": "off",
-        "unicorn/explicit-length-check": "off",
-        "unicorn/filename-case": "off",
-        "unicorn/no-await-expression-member": "off",
-        "unicorn/no-negated-condition": "off",
-        "unicorn/no-null": "off",
-        "unicorn/no-process-exit": "off",
-        "unicorn/no-useless-undefined": "off",
-        "unicorn/numeric-separators-style": "off",
-        "unicorn/prefer-module": "off",
-        "unicorn/prefer-node-protocol": "off",
-        "unicorn/prefer-ternary": "off",
-        "unicorn/prefer-top-level-await": "off",
-        "unicorn/prevent-abbreviations": "off",
-        "unicorn/switch-case-braces": "off",
-        "valid-typeof": ["error", {"requireStringLiterals": true}],
-        "yoda": "error"
-    },
-    "overrides": [
-        {
-            "files": ["web/tests/**"],
-            "rules": {
-                "no-jquery/no-selector-prop": "off"
-            }
-        },
-        {
-            "files": ["web/e2e-tests/**"],
-            "globals": {
-                "zulip_test": false
-            }
-        },
-        {
-            "files": ["web/src/**"],
-            "globals": {
-                "StripeCheckout": false
-            }
-        },
-        {
-            "files": ["**/*.ts"],
-            "extends": [
-                "plugin:@typescript-eslint/recommended",
-                "plugin:@typescript-eslint/recommended-requiring-type-checking",
-                "plugin:@typescript-eslint/strict",
-                "plugin:import/typescript"
-            ],
-            "parserOptions": {
-                "project": "tsconfig.json"
-            },
-            "settings": {
-                "import/resolver": {
-                    "node": {
-                        "extensions": [".ts", ".d.ts", ".js"] // https://github.com/import-js/eslint-plugin-import/issues/2267
-                    }
-                }
-            },
-            "globals": {
-                "JQuery": false
-            },
-            "rules": {
-                // Disable base rule to avoid conflict
-                "no-duplicate-imports": "off",
-                "no-use-before-define": "off",
-
-                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
-                "@typescript-eslint/consistent-type-imports": "error",
-                "@typescript-eslint/explicit-function-return-type": [
-                    "error",
-                    {"allowExpressions": true}
-                ],
-                "@typescript-eslint/member-ordering": "error",
-                "@typescript-eslint/no-duplicate-imports": "error",
-                "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-parameter-properties": "error",
-                "@typescript-eslint/no-unnecessary-condition": "off",
-                "@typescript-eslint/no-unnecessary-qualifier": "error",
-                "@typescript-eslint/no-unsafe-argument": "off",
-                "@typescript-eslint/no-unsafe-assignment": "off",
-                "@typescript-eslint/no-unsafe-call": "off",
-                "@typescript-eslint/no-unsafe-member-access": "off",
-                "@typescript-eslint/no-unsafe-return": "off",
-                "@typescript-eslint/no-unused-vars": ["error", {"ignoreRestSiblings": true}],
-                "@typescript-eslint/no-use-before-define": ["error", {"functions": false}],
-                "@typescript-eslint/promise-function-async": "error",
-                "import/no-cycle": "error",
-                "no-undef": "error"
-            }
-        },
-        {
-            "files": ["**/*.d.ts"],
-            "rules": {
-                "import/unambiguous": "off"
-            }
-        },
-        {
-            "files": ["web/e2e-tests/**", "web/tests/**"],
-            "globals": {
-                "CSS": false,
-                "document": false,
-                "navigator": false,
-                "window": false
-            },
-            "rules": {
-                "formatjs/no-id": "off",
-                "new-cap": "off",
-                "no-sync": "off",
-                "unicorn/prefer-prototype-methods": "off"
-            }
-        },
-        {
-            "files": ["web/debug-require.js"],
-            "env": {
-                "browser": true,
-                "es2020": false
-            },
-            "rules": {
-                // Don’t require ES features that PhantomJS doesn’t support
-                // TODO: Toggle these settings now that we don't use PhantomJS
-                "no-var": "off",
-                "object-shorthand": "off",
-                "prefer-arrow-callback": "off"
-            }
-        },
-        {
-            "files": ["web/shared/**", "web/src/**", "web/third/**"],
-            "env": {
-                "browser": true,
-                "node": false
-            },
-            "globals": {
-                "ZULIP_VERSION": false
-            },
-            "rules": {
-                "no-console": "error"
-            },
-            "settings": {
-                "import/resolver": {
-                    "webpack": {
-                        "config": "./web/webpack.config.ts"
-                    }
-                }
-            }
-        },
-        {
-            "files": ["web/shared/**"],
-            "env": {
-                "browser": false,
-                "shared-node-browser": true
-            },
-            "rules": {
-                "import/no-restricted-paths": [
-                    "error",
-                    {
-                        "zones": [
-                            {
-                                "target": "./web/shared",
-                                "from": ".",
-                                "except": ["./node_modules", "./web/shared"]
-                            }
-                        ]
-                    }
-                ],
-                "unicorn/prefer-string-replace-all": "off"
-            }
-        }
-    ]
-}

.gitattributes

@@ -1,32 +1,21 @@
-# DIFFS: Noise suppression.
-#
-# Suppress noisy generated files in diffs.
-# (When you actually want to see these diffs, use `git diff -a`.)
-
-# Large test fixtures:
-corporate/tests/stripe_fixtures/*.json -diff
-
-
-# FORMATTING
-
-# Maintain LF (Unix-style) newlines in text files.
-*   text=auto eol=lf
-
-# Make sure various media files never get somehow auto-detected as text
-# and then newline-converted.
-*.gif binary
-*.jpg binary
-*.jpeg binary
-*.eot binary
-*.woff binary
-*.woff2 binary
-*.svg binary
-*.ttf binary
-*.png binary
-*.otf binary
-*.tif binary
-*.ogg binary
-*.bson binary
-*.bmp binary
-*.mp3 binary
-*.pdf binary
+.gitignore export-ignore
+.gitattributes export-ignore
+/analytics export-ignore
+/assets export-ignore
+/bots export-ignore
+/corporate export-ignore
+/static export-ignore
+/tools export-ignore
+/zilencer export-ignore
+/templates/analytics export-ignore
+/templates/corporate export-ignore
+/templates/zilencer export-ignore
+/puppet/zulip_internal export-ignore
+/zproject/local_settings.py export-ignore
+/zproject/test_settings.py export-ignore
+/zerver/fixtures export-ignore
+/zerver/tests export-ignore
+/frontend_tests export-ignore
+/node_modules export-ignore
+/humbug export-ignore
+/locale export-ignore

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-github: zulip
-patreon: zulip
-open_collective: zulip

.github/ISSUE_TEMPLATE/1_discussed_on_czo.md

@@ -1,10 +0,0 @@
----
-name: Issue discussed in the Zulip development community
-about: Bug report, feature or improvement already discussed on chat.zulip.org.
----
-
-<!-- Issue description -->
-
-<!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
-
-CZO thread

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -1,17 +0,0 @@
----
-name: Bug report
-about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
-labels: ["bug"]
----
-
-<!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
-
-<!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
-
-**Zulip Server and web app version:**
-
-- [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 7.0+
-- [ ] Zulip Server 6.0+
-- [ ] Zulip Server 5.0 or older
-- [ ] Other or not sure

.github/ISSUE_TEMPLATE/3_feature_request.md

@@ -1,6 +0,0 @@
----
-name: Feature or improvement request
-about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
----
-
-<!-- Describe the proposal, including how it would help you or your organization. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Possible bug
-    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
-    about: Report unexpected behavior that may be a bug.
-  - name: Feature suggestion or feedback
-    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
-    about: Start a discussion about your idea for improving Zulip.
-  - name: Issue with running or upgrading a Zulip server
-    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
-    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
-  - name: Other support requests and sales questions
-    url: https://zulip.com/help/contact-support
-    about: Contact us — we're happy to help!

.github/pull_request_template.md

@@ -1,43 +0,0 @@
-<!-- Describe your pull request here.-->
-
-Fixes: <!-- Issue link, or clear description.-->
-
-<!-- If the PR makes UI changes, always include one or more still screenshots to demonstrate your changes. If it seems helpful, add a screen capture of the new functionality as well.
-
-Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
--->
-
-**Screenshots and screen captures:**
-
-<details>
-<summary>Self-review checklist</summary>
-
-<!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
-https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
-
-<!-- Once you create the PR, check off all the steps below that you have completed.
-If any of these steps are not relevant or you have not completed, leave them unchecked.-->
-
-- [ ] [Self-reviewed](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code) the changes for clarity and maintainability
-      (variable names, code reuse, readability, etc.).
-
-Communicate decisions, questions, and potential concerns.
-
-- [ ] Explains differences from previous plans (e.g., issue description).
-- [ ] Highlights technical choices and bugs encountered.
-- [ ] Calls out remaining decisions and concerns.
-- [ ] Automated tests verify logic where appropriate.
-
-Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
-
-- [ ] Each commit is a coherent idea.
-- [ ] Commit message(s) explain reasoning and motivation for changes.
-
-Completed manual review and testing of the following:
-
-- [ ] Visual appearance of the changes.
-- [ ] Responsiveness and internationalization.
-- [ ] Strings and tooltips.
-- [ ] End-to-end functionality of buttons, interactions and flows.
-- [ ] Corner cases, error conditions, and easily imagined bugs.
-</details>

.github/workflows/codeql-analysis.yml

@@ -1,40 +0,0 @@
-name: "Code scanning"
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-    branches: ["*.x", chat.zulip.org, main]
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  CodeQL:
-    permissions:
-      actions: read # for github/codeql-action/init to get workflow details
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/analyze to upload SARIF results
-    if: ${{!github.event.repository.private}}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@v3
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-
-        # Override language selection by uncommenting this and choosing your languages
-        # with:
-        #   languages: go, javascript, csharp, python, cpp, java
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2

.github/workflows/production-suite.yml

@@ -1,326 +0,0 @@
-name: Zulip production suite
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-    paths:
-      - .github/workflows/production-suite.yml
-      - "**/migrations/**"
-      - manage.py
-      - pnpm-lock.yaml
-      - puppet/**
-      - requirements/**
-      - scripts/**
-      - tools/**
-      - web/babel.config.js
-      - web/postcss.config.js
-      - web/third/**
-      - web/webpack.config.ts
-      - zerver/worker/queue_processors.py
-      - zerver/lib/push_notifications.py
-      - zerver/decorator.py
-      - zproject/**
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-permissions:
-  contents: read
-
-jobs:
-  production_build:
-    # This job builds a release tarball from the current commit, which
-    # will be used for all of the following install/upgrade tests.
-    name: Ubuntu 20.04 production build
-    runs-on: ubuntu-latest
-
-    # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-    # the top explain how to build and upload these images.
-    # Ubuntu 20.04 ships with Python 3.8.10.
-    container: zulip/ci:focal
-
-    steps:
-      - name: Add required permissions
-        run: |
-          # The checkout actions doesn't clone to ~/zulip or allow
-          # us to use the path option to clone outside the current
-          # /__w/zulip/zulip directory. Since this directory is owned
-          # by root we need to change it's ownership to allow the
-          # github user to clone the code here.
-          # Note: /__w/ is a docker volume mounted to $GITHUB_WORKSPACE
-          # which is /home/runner/work/.
-          sudo chown -R github .
-
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-      - uses: actions/checkout@v3
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore pnpm store
-        uses: actions/cache@v3
-        with:
-          path: ~/.local/share/pnpm/store
-          key: v1-pnpm-store-focal-${{ hashFiles('pnpm-lock.yaml') }}
-
-      - name: Restore python cache
-        uses: actions/cache@v3
-        with:
-          path: /srv/zulip-venv-cache
-          key: v1-venv-focal-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-focal
-
-      - name: Restore emoji cache
-        uses: actions/cache@v3
-        with:
-          path: /srv/zulip-emoji-cache
-          key: v1-emoji-focal-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
-          restore-keys: v1-emoji-focal
-
-      - name: Build production tarball
-        run: ./tools/ci/production-build
-
-      - name: Upload production build artifacts for install jobs
-        uses: actions/upload-artifact@v3
-        with:
-          name: production-tarball
-          path: /tmp/production-build
-          retention-days: 1
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
-
-  production_install:
-    # This job installs the server release tarball built above on a
-    # range of platforms, and does some basic health checks on the
-    # resulting installer Zulip server.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:focal
-            name: Ubuntu 20.04 production install and PostgreSQL upgrade with pgroonga
-            os: focal
-            extra-args: ""
-
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 production install
-            os: jammy
-            extra-args: ""
-
-          - docker_image: zulip/ci:bullseye
-            name: Debian 11 production install with custom db name and user
-            os: bullseye
-            extra-args: --test-custom-db
-
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install
-            os: bookworm
-            extra-args: ""
-
-    name: ${{ matrix.name  }}
-    container:
-      image: ${{ matrix.docker_image }}
-      options: --init
-    runs-on: ubuntu-latest
-    needs: production_build
-
-    steps:
-      - name: Download built production tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: production-tarball
-          path: /tmp
-
-      - name: Add required permissions and setup
-        run: |
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-          # Since actions/download-artifact@v2 loses all the permissions
-          # of the tarball uploaded by the upload artifact fix those.
-          chmod +x /tmp/production-upgrade-pg
-          chmod +x /tmp/production-pgroonga
-          chmod +x /tmp/production-install
-          chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore pnpm store
-        uses: actions/cache@v3
-        with:
-          path: ~/.local/share/pnpm/store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('/tmp/pnpm-lock.yaml') }}
-
-      - name: Install production
-        run: sudo /tmp/production-install ${{ matrix.extra-args }}
-
-      - name: Verify install
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Install pgroonga
-        if: ${{ matrix.os == 'focal' }}
-        run: sudo /tmp/production-pgroonga
-
-      - name: Verify install after installing pgroonga
-        if: ${{ matrix.os == 'focal' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Upgrade postgresql
-        if: ${{ matrix.os == 'focal' }}
-        run: sudo /tmp/production-upgrade-pg
-
-      - name: Verify install after upgrading postgresql
-        if: ${{ matrix.os == 'focal' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
-
-  production_upgrade:
-    # The production upgrade job starts with a container with a
-    # previous Zulip release installed, and attempts to upgrade it to
-    # the release tarball built for the current commit being tested.
-    #
-    # This is intended to catch bugs that result in the upgrade
-    # process failing.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Docker images are built from 'tools/ci/Dockerfile.prod'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:focal-3.2
-            name: 3.2 Version Upgrade
-            os: focal
-          - docker_image: zulip/ci:bullseye-4.2
-            name: 4.2 Version Upgrade
-            os: bullseye
-          - docker_image: zulip/ci:bullseye-5.0
-            name: 5.0 Version Upgrade
-            os: bullseye
-          - docker_image: zulip/ci:bullseye-6.0
-            name: 6.0 Version Upgrade
-            os: bullseye
-
-    name: ${{ matrix.name  }}
-    container:
-      image: ${{ matrix.docker_image }}
-      options: --init
-    runs-on: ubuntu-latest
-    needs: production_build
-
-    steps:
-      - name: Download built production tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: production-tarball
-          path: /tmp
-
-      - name: Add required permissions and setup
-        run: |
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-          # Since actions/download-artifact@v2 loses all the permissions
-          # of the tarball uploaded by the upload artifact fix those.
-          chmod +x /tmp/production-upgrade
-          chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Temporarily bootstrap PostgreSQL upgrades
-        # https://chat.zulip.org/#narrow/stream/43-automated-testing/topic/postgres.20client.20upgrade.20failures/near/1640444
-        # On Debian, there is an ordering issue with post-install maintainer
-        # scripts when postgresql-client-common is upgraded at the same time as
-        # postgresql-client and postgresql-client-15.  Upgrade just
-        # postgresql-client-common first, so the main upgrade process can
-        # succeed.  This is a _temporary_ work-around to improve CI signal, as
-        # the failure does represent a real failure that production systems may
-        # encounter.
-        run: sudo apt-get update && sudo apt-get install -y --only-upgrade postgresql-client-common
-
-      - name: Upgrade production
-        run: sudo /tmp/production-upgrade
-
-        # TODO: We should be running production-verify here, but it
-        # doesn't pass yet.
-        #
-        # - name: Verify install
-        #   run: sudo /tmp/production-verify
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}

.github/workflows/update-oneclick-apps.yml

@@ -1,27 +0,0 @@
-name: Update one click apps
-on:
-  release:
-    types: [published]
-permissions:
-  contents: read
-
-jobs:
-  update-digitalocean-oneclick-app:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Update DigitalOcean one click app
-        env:
-          DIGITALOCEAN_API_KEY: ${{ secrets.ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY }}
-          ZULIP_API_KEY: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_API_KEY }}
-          ZULIP_EMAIL: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_EMAIL }}
-          ZULIP_SITE: https://chat.zulip.org
-          ONE_CLICK_ACTION_STREAM: kandra ops
-          PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC: 30
-          RELEASE_VERSION: ${{ github.event.release.tag_name }}
-        run: |
-          export PATH="$HOME/.local/bin:$PATH"
-          git clone https://github.com/zulip/marketplace-partners
-          pip3 install python-digitalocean zulip fab-classic PyNaCl
-          echo $PATH
-          python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

.github/workflows/zulip-ci.yml

@@ -1,260 +0,0 @@
-# NOTE: Everything test in this file should be in `tools/test-all`.  If there's a
-# reason not to run it there, it should be there as a comment
-# explaining why.
-
-name: Zulip CI
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-permissions:
-  contents: read
-
-jobs:
-  tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Base images are built using `tools/ci/Dockerfile.prod.template`.
-          # The comments at the top explain how to build and upload these images.
-          # Ubuntu 20.04 ships with Python 3.8.10.
-          - docker_image: zulip/ci:focal
-            name: Ubuntu 20.04 (Python 3.8, backend + frontend)
-            os: focal
-            include_documentation_tests: false
-            include_frontend_tests: true
-          # Debian 11 ships with Python 3.9.2.
-          - docker_image: zulip/ci:bullseye
-            name: Debian 11 (Python 3.9, backend + documentation)
-            os: bullseye
-            include_documentation_tests: true
-            include_frontend_tests: false
-          # Ubuntu 22.04 ships with Python 3.10.4.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 (Python 3.10, backend)
-            os: jammy
-            include_documentation_tests: false
-            include_frontend_tests: false
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend)
-            os: bookworm
-            include_documentation_tests: false
-            include_frontend_tests: false
-
-    runs-on: ubuntu-latest
-    name: ${{ matrix.name }}
-    container: ${{ matrix.docker_image }}
-    env:
-      # GitHub Actions sets HOME to /github/home which causes
-      # problem later in provision and frontend test that runs
-      # tools/setup/postgresql-init-dev-db because of the .pgpass
-      # location. PostgreSQL (psql) expects .pgpass to be at
-      # /home/github/.pgpass and setting home to `/home/github/`
-      # ensures it written there because we write it to ~/.pgpass.
-      HOME: /home/github/
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore pnpm store
-        uses: actions/cache@v3
-        with:
-          path: ~/.local/share/pnpm/store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
-
-      - name: Restore python cache
-        uses: actions/cache@v3
-        with:
-          path: /srv/zulip-venv-cache
-          key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-${{ matrix.os }}
-
-      - name: Restore emoji cache
-        uses: actions/cache@v3
-        with:
-          path: /srv/zulip-emoji-cache
-          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json', 'tools/setup/emoji/build_emoji', 'tools/setup/emoji/emoji_setup_utils.py', 'tools/setup/emoji/emoji_names.py', 'package.json') }}
-          restore-keys: v1-emoji-${{ matrix.os }}
-
-      - name: Install dependencies
-        run: |
-          # This is the main setup job for the test suite
-          ./tools/ci/setup-backend --skip-dev-db-build
-          scripts/lib/clean_unused_caches.py --verbose --threshold=0
-
-      - name: Run tools test
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-tools
-
-      - name: Run Codespell lint
-        run: |
-          source tools/ci/activate-venv
-          ./tools/run-codespell
-
-      - name: Run backend lint
-        run: |
-          source tools/ci/activate-venv
-          echo "Test suite is running under $(python --version)."
-          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
-
-      - name: Run frontend lint
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
-
-      - name: Run backend tests
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-backend --coverage --xml-report --no-html-report --include-webhooks --no-cov-cleanup --ban-console-output
-
-      - name: Run mypy
-        run: |
-          source tools/ci/activate-venv
-          # We run mypy after the backend tests so we get output from the
-          # backend tests, which tend to uncover more serious problems, first.
-          ./tools/run-mypy --version
-          ./tools/run-mypy
-
-      - name: Run miscellaneous tests
-        run: |
-          source tools/ci/activate-venv
-
-          # Currently our compiled requirements files will differ for different
-          # Python versions, so we will run test-locked-requirements only on the
-          # platform with the oldest one.
-          # ./tools/test-locked-requirements
-          # ./tools/test-run-dev  # https://github.com/zulip/zulip/pull/14233
-          #
-          # This test has been persistently flaky at like 1% frequency, is slow,
-          # and is for a very specific single feature, so we don't run it by default:
-          # ./tools/test-queue-worker-reload
-
-          ./tools/test-migrations
-          ./tools/setup/optimize-svg --check
-          ./tools/setup/generate_integration_bots_avatars.py --check-missing
-          ./tools/ci/check-executables
-
-          # Ban check-database-compatibility from transitively
-          # relying on static/generated, because it might not be
-          # up-to-date at that point in upgrade-zulip-stage-2.
-          chmod 000 static/generated web/generated
-          ./scripts/lib/check-database-compatibility
-          chmod 755 static/generated web/generated
-
-      - name: Run documentation and api tests
-        if: ${{ matrix.include_documentation_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # In CI, we only test links we control in test-documentation to avoid flakes
-          ./tools/test-documentation --skip-external-links
-          ./tools/test-help-documentation --skip-external-links
-          ./tools/test-api
-
-      - name: Run node tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Run the node tests first, since they're fast and deterministic
-          ./tools/test-js-with-node --coverage --parallel=1
-
-      - name: Check schemas
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Check that various schemas are consistent. (is fast)
-          ./tools/check-schemas
-
-      - name: Check capitalization of strings
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./manage.py makemessages --locale en
-          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
-          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
-
-      - name: Run puppeteer tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-js-with-puppeteer
-
-      - name: Check pnpm dedupe
-        if: ${{ matrix.include_frontend_tests }}
-        run: pnpm dedupe --check
-
-      - name: Check for untracked files
-        run: |
-          source tools/ci/activate-venv
-          # This final check looks for untracked files that may have been
-          # created by test-backend or provision.
-          untracked="$(git ls-files --exclude-standard --others)"
-          if [ -n "$untracked" ]; then
-              printf >&2 "Error: untracked files:\n%s\n" "$untracked"
-              exit 1
-          fi
-
-      - name: Test locked requirements
-        if: ${{ matrix.os == 'focal' }}
-        run: |
-          . /srv/zulip-py3-venv/bin/activate && \
-          ./tools/test-locked-requirements
-
-      - name: Upload coverage reports
-
-        # Only upload coverage when both frontend and backend
-        # tests are run.
-        if: ${{ matrix.include_frontend_tests }}
-        uses: codecov/codecov-action@v3
-        with:
-          files: var/coverage.xml,var/node-coverage/lcov.info
-
-      - name: Store Puppeteer artifacts
-        # Upload these on failure, as well
-        if: ${{ always() && matrix.include_frontend_tests }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: puppeteer
-          path: ./var/puppeteer
-          retention-days: 60
-
-      - name: Check development database build
-        run: ./tools/ci/setup-backend
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}

.gitignore

@@ -1,88 +1,26 @@
-# Quick format and style primer:
-#
-#  * If a pattern is meant only for a specific location, it should have a
-#    leading slash, like `/staticfiles.json`.
-#    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
-#      will do, but this makes a confusing pattern.  Adding a leading slash
-#      is clearer.
-#
-#  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
-#    match in any subdirectory.
-#
-#  * Subdirectories with several internal things to ignore get their own
-#    `.gitignore` files.
-#
-#  * Comments must be on their own line.  (Otherwise they don't work.)
-#
-# See `git help ignore` for details on the format.
-
-## Config files for the dev environment
-/zproject/dev-secrets.conf
-/tools/conf.ini
-/tools/custom_provision
-/tools/droplets/conf.ini
-
-## Byproducts of setting up and using the dev environment
 *.pyc
-package-lock.json
-
-/.vagrant
-/var/*
-!/var/puppeteer
-/var/puppeteer/*
-!/var/puppeteer/test_credentials.d.ts
-
-/.dmypy.json
-/.ruff_cache
-
-# Generated i18n data
-/locale/en
-/locale/language_options.json
-/locale/language_name_map.json
-/locale/*/mobile.json
-
-# Static build
-*.mo
-npm-debug.log
-/.pnpm-store
-/node_modules
-/prod-static
-/staticfiles.json
-/webpack-stats-production.json
-zulip-git-version
-
-# Test / analysis tools
-.coverage
-
-## Files (or really symlinks) created in a prod deployment
-/zproject/prod_settings.py
-/zulip-current-venv
-/zulip-py3-venv
-
-## Files left by various editors and local environments
-# (Ideally these should be in everyone's respective personal gitignore files.)
 *~
+/prod-static
+/errors/*
 *.sw[po]
-.idea
+*.DS_Store
+stats/
 .kdev4
+.idea
 zulip.kdev4
+coverage/
+.coverage
+/queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
-*.sublime-project
-*.sublime-workspace
-*.DS_Store
-# VS Code. Avoid checking in .vscode in general, while still specifying
-# recommended extensions for working with this repository.
-/.vscode/**/*
-!/.vscode/extensions.json
-# .cache/ is generated by VS Code test runner
-.cache/
-.eslintcache
-
-# Core dump files
-core
-
-## Miscellaneous
-# (Ideally this section is empty.)
-.transifexrc
+.vagrant
+/zproject/dev-secrets.conf
+static/js/bundle.js
+static/third/gemoji/
+static/third/zxcvbn/
+static/locale/language_options.json
+node_modules
+npm-debug.log
+*.mo
+var/*

.gitlint

@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing
-
-extra-path=tools/lib/gitlint_rules.py
-
-[title-match-regex]
-regex=^(.+:\ )?[A-Z].+\.$
-
-[title-max-length]
-line-length=72
-
-[body-max-line-length]
-line-length=76

.mailmap

@@ -1,131 +0,0 @@
-# This file teaches `git log` and friends the canonical names
-# and email addresses to use for our contributors.
-#
-# For details on the format, see:
-#   https://git.github.io/htmldocs/gitmailmap.html
-#
-# Handy commands for examining or adding to this file:
-#
-#     # shows all names/emails after mapping, sorted:
-#   $ git shortlog -es | sort -k2
-#
-#     # shows raw names/emails, filtered by mapped name:
-#   $ git log --format='%an %ae' --author=$NAME | uniq -c
-
-acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
-Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
-Adam Benesh <Adam.Benesh@gmail.com>
-Adarsh Tiwari <xoldyckk@gmail.com>
-Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
-Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
-Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
-Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
-Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
-Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
-Aman Agrawal <amanagr@zulip.com>
-Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
-Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
-Aryan Shridhar <aryanshridhar7@gmail.com>
-aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
-Austin Riba <austin@zulip.com> <austin@m51.io>
-BIKI DAS <bikid475@gmail.com>
-Brijmohan Siyag <brijsiyag@gmail.com>
-Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
-Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
-Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
-Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
-Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dinesh <chdinesh1089@gmail.com>
-Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
-Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
-Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
-Evy Kassirer <evy.kassirer@gmail.com>
-Evy Kassirer <evy.kassirer@gmail.com> <evykassirer@users.noreply.github.com>
-Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
-Greg Price <greg@zulip.com> <gnprice@gmail.com>
-Greg Price <greg@zulip.com> <greg@zulipchat.com>
-Greg Price <greg@zulip.com> <price@mit.edu>
-Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
-Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
-Jai soni <jai_s@me.iitr.ac.in>
-Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
-Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
-Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
-Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
-Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
-Karl Stolley <karl@zulip.com> <karl@stolley.dev>
-Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
-Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
-Kevin Scott <kevin.scott.98@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
-Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
-Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
-Matt Keller <matt@zulip.com>
-Matt Keller <matt@zulip.com> <m@cognusion.com>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
-Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
-nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
-Palash Baderia <palash.baderia@outlook.com>
-Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
-Palash Raghuwanshi <singhpalash0@gmail.com>
-Parth <mittalparth22@gmail.com>
-Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
-Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
-Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
-Rein Zustand (rht) <rhtbot@protonmail.com>
-Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
-Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
-Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
-Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
-Rixant Rokaha <rixantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
-Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Satyam Bansal <sbansal1999@gmail.com>
-Sayam Samal <samal.sayam@gmail.com>
-Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
-Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
-Shlok Patel <shlokcpatel2001@gmail.com>
-Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
-Steve Howell <showell@zulip.com> <showell30@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@zulipchat.com>
-Steve Howell <showell@zulip.com> <steve@humbughq.com>
-Steve Howell <showell@zulip.com> <steve@zulip.com>
-strifel <info@strifel.de>
-Tim Abbott <tabbott@zulip.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
-Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
-Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
-Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
-Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
-umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
-umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com>
-Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
-Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
-Zev Benjamin <zev@zulip.com> <zev@mit.edu>
-Zixuan James Li <p359101898@gmail.com>
-Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>
-Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
-Joseph Ho <josephho678@gmail.com>
-Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>

.npmrc

@@ -1 +0,0 @@
-ignore-dep-scripts=true

.prettierignore

@@ -1,11 +0,0 @@
-pnpm-lock.yaml
-/api_docs/**/*.md
-/corporate/tests/stripe_fixtures
-/help/**/*.md
-/locale
-/templates/**/*.md
-/tools/setup/emoji/emoji_map.json
-/web/third
-/zerver/tests/fixtures
-/zerver/webhooks/*/doc.md
-/zerver/webhooks/*/fixtures

.pyre_configuration

@@ -1,15 +0,0 @@
-{
-  "source_directories": ["."],
-  "taint_models_path": [
-      "stubs/taint",
-      "zulip-py3-venv/lib/pyre_check/taint/"
-  ],
-  "search_path": [
-      "stubs/",
-      "zulip-py3-venv/lib/pyre_check/stubs/"
-  ],
-  "typeshed": "zulip-py3-venv/lib/pyre_check/typeshed/",
-  "exclude": [
-      "/srv/zulip/zulip-py3-venv/.*"
-  ]
-}

.readthedocs.yaml

@@ -1,15 +0,0 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
-
-python:
-  install:
-    - requirements: requirements/docs.txt

.sonarcloud.properties

@@ -1 +0,0 @@
-sonar.inclusions=**/*.py,**/*.html

.travis.yml

@@ -0,0 +1,47 @@
+dist: trusty
+before_install:
+   - nvm install 0.10
+install:
+  - pip install coveralls
+  - tools/travis/setup-$TEST_SUITE
+  - tools/clean-venv-cache --travis
+cache:
+  - apt: false
+  - directories:
+    - $HOME/phantomjs
+    - $HOME/zulip-venv-cache
+    - node_modules
+    - $HOME/node
+env:
+  global:
+    - COVERALLS_PARALLEL=true
+    - COVERALLS_SERVICE_NAME=travis-pro
+    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
+    - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
+language: python
+python:
+  - "2.7"
+  - "3.4"
+matrix:
+  include:
+    - python: "3.4"
+      env: TEST_SUITE=static-analysis
+    - python: "2.7"
+      env: TEST_SUITE=production
+      sudo: required
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
+sudo: required
+services:
+- docker
+addons:
+  postgresql: "9.3"
+after_success:
+  coveralls
+notifications:
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN

.tx/config

@@ -1,40 +1,17 @@
-# Migrated from transifex-client format with `tx migrate`
-#
-# See https://developers.transifex.com/docs/using-the-client which hints at
-# this format, but in general, the headings are in the format of:
-#
-# [o:<org>:p:<project>:r:<resource>]
-
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
-[o:zulip:p:zulip:r:djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
+[zulip.djangopo]
+source_file = static/locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
+file_filter = static/locale/<lang>/LC_MESSAGES/django.po
+lang_map = zh-Hans: zh_CN
 
-[o:zulip:p:zulip:r:mobile]
-file_filter = locale/<lang>/mobile.json
-source_file = locale/en/mobile.json
+[zulip.translationsjson]
+source_file = static/locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
+file_filter = static/locale/<lang>/translations.json
+lang_map = zh-Hans: zh-CN
 
-[o:zulip:p:zulip:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip-test:r:djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
-source_lang = en
-type = PO
-
-[o:zulip:p:zulip-test:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON

.vscode/extensions.json

@@ -1,23 +0,0 @@
-{
-    // Recommended VS Code extensions for zulip/zulip.
-    //
-    // VS Code prompts a user to install the recommended extensions
-    // when a workspace is opened for the first time.  The user can
-    // also review the list with the 'Extensions: Show Recommended
-    // Extensions' command.  See
-    // https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions
-    // for more information.
-    //
-    // Extension identifier format: ${publisher}.${name}.
-    // Example: vscode.csharp
-
-    "recommendations": [
-        "42crunch.vscode-openapi",
-        "dbaeumer.vscode-eslint",
-        "esbenp.prettier-vscode",
-        "ms-vscode-remote.vscode-remote-extensionpack"
-    ],
-
-    // Extensions recommended by VS Code which are not recommended for users of zulip/zulip.
-    "unwantedRecommendations": []
-}

CODE_OF_CONDUCT.md

@@ -1,172 +0,0 @@
-# Zulip Code of Conduct
-
-Like the technical community as a whole, the Zulip team and community is
-made up of a mixture of professionals and volunteers from all over the
-world, working on every aspect of the mission, including mentorship,
-teaching, and connecting people.
-
-Diversity is one of our huge strengths, but it can also lead to
-communication issues and unhappiness. To that end, we have a few ground
-rules that we ask people to adhere to. This code applies equally to
-founders, mentors, and those seeking help and guidance.
-
-This isn't an exhaustive list of things that you can't do. Rather, take it
-in the spirit in which it's intended --- a guide to make it easier to enrich
-all of us and the technical communities in which we participate.
-
-## Expected behavior
-
-The following behaviors are expected and requested of all community members:
-
-- Participate. In doing so, you contribute to the health and longevity of
-  the community.
-- Exercise consideration and respect in your speech and actions.
-- Attempt collaboration before conflict. Assume good faith.
-- Refrain from demeaning, discriminatory, or harassing behavior and speech.
-- Take action or alert community leaders if you notice a dangerous
-  situation, someone in distress, or violations of this code, even if they
-  seem inconsequential.
-- Community event venues may be shared with members of the public; be
-  respectful to all patrons of these locations.
-
-## Unacceptable behavior
-
-The following behaviors are considered harassment and are unacceptable
-within the Zulip community:
-
-- Jokes or derogatory language that singles out members of any race,
-  ethnicity, culture, national origin, color, immigration status, social and
-  economic class, educational level, language proficiency, sex, sexual
-  orientation, gender identity and expression, age, size, family status,
-  political belief, religion, and mental and physical ability.
-- Violence, threats of violence, or violent language directed against
-  another person.
-- Disseminating or threatening to disseminate another person's personal
-  information.
-- Personal insults of any sort.
-- Posting or displaying sexually explicit or violent material.
-- Inappropriate photography or recording.
-- Deliberate intimidation, stalking, or following (online or in person).
-- Unwelcome sexual attention. This includes sexualized comments or jokes,
-  inappropriate touching or groping, and unwelcomed sexual advances.
-- Sustained disruption of community events, including talks and
-  presentations.
-- Advocating for, or encouraging, any of the behaviors above.
-
-## Reporting and enforcement
-
-Harassment and other code of conduct violations reduce the value of the
-community for everyone. If someone makes you or anyone else feel unsafe or
-unwelcome, please report it to the community organizers at
-zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
-report either personally or anonymously.
-
-If a community member engages in unacceptable behavior, the community
-organizers may take any action they deem appropriate, up to and including a
-temporary ban or permanent expulsion from the community without warning (and
-without refund in the case of a paid event).
-
-If someone outside the development community (e.g. a user of the Zulip
-software) engages in unacceptable behavior that affects someone in the
-community, we still want to know. Even if we don't have direct control over
-the violator, the community organizers can still support the people
-affected, reduce the chance of a similar violation in the future, and take
-any direct action we can.
-
-The nature of reporting means it can only help after the fact. If you see
-something you can do while a violation is happening, do it. A lot of the
-harms of harassment and other violations can be mitigated by the victim
-knowing that the other people present are on their side.
-
-All reports will be kept confidential. In some cases, we may determine that a
-public statement will need to be made. In such cases, the identities of all
-victims and reporters will remain confidential unless those individuals
-instruct us otherwise.
-
-## Scope
-
-We expect all community participants (contributors, paid or otherwise,
-sponsors, and other guests) to abide by this Code of Conduct in all
-community venues, online and in-person, as well as in all private
-communications pertaining to community business.
-
-This Code of Conduct and its related procedures also applies to unacceptable
-behavior occurring outside the scope of community activities when such
-behavior has the potential to adversely affect the safety and well-being of
-community members.
-
-## License and attribution
-
-This Code of Conduct is adapted from the
-[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
-under a
-[Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
-license.
-
-## Moderating the Zulip community
-
-Anyone can help moderate the Zulip community by helping make sure that folks are
-aware of the [community guidelines](https://zulip.com/development-community/)
-and this Code of Conduct, and that we maintain a positive and respectful
-atmosphere.
-
-Here are some guidelines for you how can help:
-
-- Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
-  and just trying to keep the atmosphere warm make the whole community function
-  more smoothly. New participants who feel accepted, listened to and respected
-  are likely to treat others the same way.
-
-- Be familiar with the [community
-  guidelines](https://zulip.com/development-community/), and cite them liberally
-  when a user violates them. Be polite but firm. Some examples:
-
-  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
-    you ask a question. As noted in the [guidelines for this
-    community](https://zulip.com/development-community/):
-
-    > Use @-mentions sparingly… there is generally no need to @-mention a
-    > core contributor unless you need their timely attention.
-
-  - @user, please keep in mind the following [community
-    guideline](https://zulip.com/development-community/):
-
-    > Don’t ask the same question in multiple places. Moderators read every
-    > public stream, and make sure every question gets a reply.
-
-    I’ve gone ahead and moved the other copy of this message to this thread.
-
-  - If asked a question in a PM that is better discussed in a public stream:
-    > Hi @user! Please start by reviewing
-    > https://zulip.com/development-community/#community-norms to learn how to
-    > get help in this community.
-
-- Users sometimes think chat.zulip.org is a testing instance. When this happens,
-  kindly direct them to use the **#test here** stream.
-
-- If you see a message that’s posted in the wrong place, go ahead and move it if
-  you have permissions to do so, even if you don’t plan to respond to it.
-  Leaving the “Send automated notice to new topic” option enabled helps make it
-  clear what happened to the person who sent the message.
-
-  If you are responding to a message that's been moved, mention the user in your
-  reply, so that the mention serves as a notification of the new location for
-  their conversation.
-
-- If a user is posting spam, please report it to an administrator. They will:
-
-  - Change the user's name to `<name> (spammer)` and deactivate them.
-  - Delete any spam messages they posted in public streams.
-
-- We care very much about maintaining a respectful tone in our community. If you
-  see someone being mean or rude, point out that their tone is inappropriate,
-  and ask them to communicate their perspective in a respectful way in the
-  future. If you don’t feel comfortable doing so yourself, feel free to ask a
-  member of Zulip's core team to take care of the situation.
-
-- Try to assume the best intentions from others (given the range of
-  possibilities presented by their visible behavior), and stick with a friendly
-  and positive tone even when someone‘s behavior is poor or disrespectful.
-  Everyone has bad days and stressful situations that can result in them
-  behaving not their best, and while we should be firm about our community
-  rules, we should also enforce them with kindness.

CONTRIBUTING.md

@@ -1,390 +0,0 @@
-# Contributing guide
-
-Welcome to the Zulip community!
-
-## Zulip development community
-
-The primary communication forum for the Zulip community is the Zulip
-server hosted at [chat.zulip.org](https://chat.zulip.org/):
-
-- **Users** and **administrators** of Zulip organizations stop by to
-  ask questions, offer feedback, and participate in product design
-  discussions.
-- **Contributors to the project**, including the **core Zulip
-  development team**, discuss ongoing and future projects, brainstorm
-  ideas, and generally help each other out.
-
-Everyone is welcome to [sign up](https://chat.zulip.org/) and
-participate — we love hearing from our users! Public streams in the
-community receive thousands of messages a week. We recommend signing
-up using the special invite links for
-[users](https://chat.zulip.org/join/t5crtoe62bpcxyisiyglmtvb/),
-[self-hosters](https://chat.zulip.org/join/wnhv3jzm6afa4raenedanfno/)
-and
-[contributors](https://chat.zulip.org/join/npzwak7vpmaknrhxthna3c7p/)
-to get a curated list of initial stream subscriptions.
-
-To learn how to get started participating in the community, including [community
-norms](https://zulip.com/development-community/#community-norms) and [where to
-post](https://zulip.com/development-community/#where-do-i-send-my-message),
-check out our [Zulip development community
-guide](https://zulip.com/development-community/). The Zulip community is
-governed by a [code of
-conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
-
-## Ways to contribute
-
-To make a code or documentation contribution, read our
-[step-by-step guide](#your-first-codebase-contribution) to getting
-started with the Zulip codebase. A small sample of the type of work that
-needs doing:
-
-- Bug squashing and feature development on our Python/Django
-  [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip), React Native
-  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
-  [desktop app](https://github.com/zulip/zulip-desktop).
-- Building out our
-  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
-- [Writing an integration](https://zulip.com/api/integrations-overview).
-- Improving our [user](https://zulip.com/help/) or
-  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
-- [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
-  and manually testing pull requests.
-
-**Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. For example, you can:
-
-- Report issues, including both [feature
-  requests](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html)
-  and [bug
-  reports](https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html).
-- [Give feedback](#user-feedback) if you are evaluating or using Zulip.
-- [Participate
-  thoughtfully](https://zulip.readthedocs.io/en/latest/contributing/design-discussions.html)
-  in design discussions.
-- [Sponsor Zulip](https://github.com/sponsors/zulip) through the GitHub sponsors program.
-- [Translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip into your language.
-- [Stay connected](#stay-connected) with Zulip, and [help others
-  find us](#help-others-find-zulip).
-
-## Your first codebase contribution
-
-This section has a step by step guide to starting as a Zulip codebase
-contributor. It's long, but don't worry about doing all the steps perfectly;
-no one gets it right the first time, and there are a lot of people available
-to help.
-
-- First, make an account on the
-  [Zulip community server](https://zulip.com/development-community/),
-  paying special attention to the
-  [community norms](https://zulip.com/development-community/#community-norms).
-  If you'd like, introduce yourself in
-  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
-  your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken or interesting/helpful as you
-  started using the product.
-- Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-- [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-  getting help in
-  [#provision help](https://chat.zulip.org/#narrow/stream/21-provision-help)
-  if you run into any troubles.
-- Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
-- Go through the [new application feature
-  tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
-  how the Zulip codebase is organized and how to find code in it.
-- Read the [Zulip guide to
-  Git](https://zulip.readthedocs.io/en/latest/git/index.html) if you
-  are unfamiliar with Git or Zulip's rebase-based Git workflow,
-  getting help in [#git
-  help](https://chat.zulip.org/#narrow/stream/44-git-help) if you run
-  into any troubles. Even Git experts should read the [Zulip-specific
-  Git tools
-  page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
-
-### Where to look for an issue
-
-Now you're ready to pick your first issue! Zulip has several repositories you
-can check out, depending on your interests. There are hundreds of open issues in
-the [main Zulip server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-alone.
-
-You can look through issues tagged with the "help wanted" label, which is used
-to indicate the issues that are ready for contributions. Some repositories also
-use the "good first issue" label to tag issues that are especially approachable
-for new contributors.
-
-- [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- [Mobile apps](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
-- [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-
-### Picking an issue to work on
-
-There's a lot to learn while making your first pull request, so start small!
-Many first contributions have fewer than 10 lines of changes (not counting
-changes to tests).
-
-We recommend the following process for finding an issue to work on:
-
-1. Read the description of an issue tagged with the "help wanted" label and make
-   sure you understand it.
-2. If it seems promising, poke around the product
-   (on [chat.zulip.org](https://chat.zulip.org) or in the development
-   environment) until you know how the piece being
-   described fits into the bigger picture. If after some exploration the
-   description seems confusing or ambiguous, post a question on the GitHub
-   issue, as others may benefit from the clarification as well.
-3. When you find an issue you like, try to get started working on it. See if you
-   can find the part of the code you'll need to modify (`git grep` is your
-   friend!) and get some idea of how you'll approach the problem.
-4. If you feel lost, that's OK! Go through these steps again with another issue.
-   There's plenty to work on, and the exploration you do will help you learn
-   more about the project.
-
-Note that you are _not_ claiming an issue while you are iterating through steps
-1-4. _Before you claim an issue_, you should be confident that you will be able to
-tackle it effectively.
-
-Additional tips for the [main server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
-
-- We especially recommend browsing recently opened issues, as there are more
-  likely to be easy ones for you to find.
-- All issues are partitioned into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-- Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are highly confident you can resolve the
-  issue correctly and completely.
-
-### Claiming an issue
-
-#### In the main server/web app repository and Zulip Terminal repository
-
-The Zulip server/web app repository
-([`zulip/zulip`](https://github.com/zulip/zulip/)) and the Zulip Terminal
-repository ([`zulip/zulip-terminal`](https://github.com/zulip/zulip-terminal/))
-are set up with a GitHub workflow bot called
-[Zulipbot](https://github.com/zulip/zulipbot), which manages issues and pull
-requests in order to create a better workflow for Zulip contributors.
-
-To claim an issue in these repositories, simply post a comment that says
-`@zulipbot claim` to the issue thread. If the issue is tagged with a [help
-wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-label, Zulipbot will immediately assign the issue to you.
-
-Note that new contributors can only claim one issue until their first pull request is
-merged. This is to encourage folks to finish ongoing work before starting
-something new. If you would like to pick up a new issue while waiting for review
-on an almost-ready pull request, you can post a comment to this effect on the
-issue you're interested in.
-
-#### In other Zulip repositories
-
-There is no bot for other Zulip repositories
-([`zulip/zulip-mobile`](https://github.com/zulip/zulip-mobile/), etc.). If
-you are interested in claiming an issue in one of these repositories, simply
-post a comment on the issue thread saying that you'd like to work on it. There
-is no need to @-mention the issue creator in your comment.
-
-Please follow the same guidelines as described above: find an issue labeled
-"help wanted", and only pick up one issue at a time to start with.
-
-### Working on an issue
-
-You're encouraged to ask questions on how to best implement or debug your
-changes -- the Zulip maintainers are excited to answer questions to help you
-stay unblocked and working efficiently. You can ask questions in the [Zulip
-development community](https://zulip.com/development-community/), or on the
-GitHub issue or pull request.
-
-To get early feedback on any UI changes, we encourage you to post screenshots of
-your work in the [#design
-stream](https://chat.zulip.org/#narrow/stream/101-design) in the [Zulip
-development community](https://zulip.com/development-community/)
-
-For more advice, see [What makes a great Zulip
-contributor?](#what-makes-a-great-zulip-contributor) below. It's OK if your
-first issue takes you a while; that's normal! You'll be able to work a lot
-faster as you build experience.
-
-### Submitting a pull request
-
-See the [pull request review
-process](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-guide for detailed instructions on how to submit a pull request, and information
-on the stages of review your PR will go through.
-
-### Beyond the first issue
-
-To find a second issue to work on, we recommend looking through issues with the same
-`area:` label as the last issue you resolved. You'll be able to reuse the
-work you did learning how that part of the codebase works. Also, the path to
-becoming a core developer often involves taking ownership of one of these area
-labels.
-
-### Common questions
-
-- **What if somebody is already working on the issue I want do claim?** There
-  are lots of issue to work on! If somebody else is actively working on the
-  issue, you can find a different one, or help with
-  reviewing their work.
-- **What if somebody else claims an issue while I'm figuring out whether or not to
-  work on it?** No worries! You can contribute by providing feedback on
-  their pull request. If you've made good progress in understanding part of the
-  codebase, you can also find another "help wanted" issue in the same area to
-  work on.
-- **What if there is already a pull request for the issue I want to work on?**
-  Start by reviewing the existing work. If you agree with the approach, you can
-  use the existing pull request (PR) as a starting point for your contribution. If
-  you think a different approach is needed, you can post a new PR, with a comment that clearly
-  explains _why_ you decided to start from scratch.
-- **Can I come up with my own feature idea and work on it?** We welcome
-  suggestions of features or other improvements that you feel would be valuable. If you
-  have a new feature you'd like to add, you can start a conversation [in our
-  development community](https://zulip.com/development-community/#where-do-i-send-my-message)
-  explaining the feature idea and the problem that you're hoping to solve.
-- **I'm waiting for the next round of review on my PR. Can I pick up
-  another issue in the meantime?** Someone's first Zulip PR often
-  requires quite a bit of iteration, so please [make sure your pull
-  request is reviewable][reviewable-pull-requests] and go through at
-  least one round of feedback from others before picking up a second
-  issue. After that, sure! If
-  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
-  claim an issue, you can post a comment describing the status of your
-  other work on the issue you're interested in, and asking for the
-  issue to be assigned to you. Note that addressing feedback on
-  in-progress PRs should always take priority over starting a new PR.
-- **I think my PR is done, but it hasn't been merged yet. What's going on?**
-  1. **Double-check that you have addressed all the feedback**, including any comments
-     on [Git commit
-     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
-  2. If all the feedback has been addressed, did you [leave a
-     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
-     explaining that you have done so and **requesting another review**? If not,
-     it may not be clear to project maintainers or reviewers that your PR is
-     ready for another look.
-  3. There may be a pause between initial rounds of review for your PR and final
-     review by project maintainers. This is normal, and we encourage you to **work
-     on other issues** while you wait.
-  4. If you think the PR is ready and haven't seen any updates for a couple
-     of weeks, it can be helpful to **leave another comment**. Summarize the
-     overall state of the review process and your work, and indicate that you
-     are waiting for a review.
-  5. Finally, **Zulip project maintainers are people too**! They may be busy
-     with other work, and sometimes they might even take a vacation. ;) It can
-     occasionally take a few weeks for a PR in the final stages of the review
-     process to be merged.
-
-[reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
-
-## What makes a great Zulip contributor?
-
-Zulip has a lot of experience working with new contributors. In our
-experience, these are the best predictors of success:
-
-- [Asking great questions][great-questions]. It's very hard to answer a general
-  question like, "How do I do this issue?" When asking for help, explain your
-  current understanding, including what you've done or tried so far and where
-  you got stuck. Post tracebacks or other error messages if appropriate. For
-  more advice, check out [our guide][great-questions]!
-- Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
-- Submitting carefully tested code. See our [detailed guide on how to review
-  code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
-  (yours or someone else's).
-- Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-- Working to [make your pull requests easy to
-  review](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html).
-- Clearly describing what you have implemented and why. For example, if your
-  implementation differs from the issue description in some way or is a partial
-  step towards the requirements described in the issue, be sure to call
-  out those differences.
-- Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-- Being helpful and friendly on the [Zulip community
-  server](https://zulip.com/development-community/).
-
-[great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
-
-## User feedback
-
-Nearly every feature we develop starts with a user request. If you are part
-of a group that is either using or considering using Zulip, we would love to
-hear about your experience with the product. If you're not sure what to
-write, here are some questions we're always very curious to know the answer
-to:
-
-- Evaluation: What is the process by which your organization chose or will
-  choose a group chat product?
-- Pros and cons: What are the pros and cons of Zulip for your organization,
-  and the pros and cons of other products you are evaluating?
-- Features: What are the features that are most important for your
-  organization? In the best-case scenario, what would your chat solution do
-  for you?
-- Onboarding: If you remember it, what was your impression during your first
-  few minutes of using Zulip? What did you notice, and how did you feel? Was
-  there anything that stood out to you as confusing, or broken, or great?
-- Organization: What does your organization do? How big is the organization?
-  A link to your organization's website?
-
-You can contact us in the [#feedback stream of the Zulip development
-community](https://chat.zulip.org/#narrow/stream/137-feedback) or
-by emailing [support@zulip.com](mailto:support@zulip.com).
-
-## Outreach programs
-
-Zulip regularly participates in [Google Summer of Code
-(GSoC)](https://developers.google.com/open-source/gsoc/) and
-[Outreachy](https://www.outreachy.org/). We have been a GSoC mentoring
-organization since 2016, and we accept 15-20 GSoC participants each summer. In
-the past, we’ve also participated in [Google
-Code-In](https://developers.google.com/open-source/gci/), and hosted summer
-interns from Harvard, MIT, and Stanford.
-
-Check out our [outreach programs
-overview](https://zulip.readthedocs.io/en/latest/outreach/overview.html) to learn
-more about participating in an outreach program with Zulip. Most of our program
-participants end up sticking around the project long-term, and many have become
-core team members, maintaining important parts of the project. We hope you
-apply!
-
-## Stay connected
-
-Even if you are not logging into the development community on a regular basis,
-you can still stay connected with the project.
-
-- Follow us [on Twitter](https://twitter.com/zulip).
-- Subscribe to [our blog](https://blog.zulip.org/).
-- Join or follow the project [on LinkedIn](https://www.linkedin.com/company/zulip-project/).
-
-## Help others find Zulip
-
-Here are some ways you can help others find Zulip:
-
-- Star us on GitHub. There are four main repositories:
-  [server/web](https://github.com/zulip/zulip),
-  [mobile](https://github.com/zulip/zulip-mobile),
-  [desktop](https://github.com/zulip/zulip-desktop), and
-  [Python API](https://github.com/zulip/python-zulip-api).
-
-- "Like" and retweet [our tweets](https://twitter.com/zulip).
-
-- Upvote and post feedback on Zulip on comparison websites. A couple specific
-  ones to highlight:
-
-  - [AlternativeTo](https://alternativeto.net/software/zulip-chat-server/). You can also
-    [upvote Zulip](https://alternativeto.net/software/slack/) on their page
-    for Slack.
-  - [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
-    it, and upvote the reasons why people like Zulip that you find most
-    compelling.

Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+WORKDIR /srv/zulip

Dockerfile-postgresql

@@ -1,25 +0,0 @@
-# This is a multiarch Dockerfile.  See https://docs.docker.com/desktop/multi-arch/
-#
-# To set up the first time:
-#     docker buildx create --name multiarch --use
-#
-# To build:
-#     docker buildx build --platform linux/amd64,linux/arm64 \
-#       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
-
-# Currently the PostgreSQL images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image can not currently be upgraded
-# without users needing a manual pgdump and restore.
-
-# https://hub.docker.com/r/groonga/pgroonga/tags
-ARG PGROONGA_VERSION=latest
-ARG POSTGRESQL_VERSION=14
-FROM groonga/pgroonga:$PGROONGA_VERSION-alpine-$POSTGRESQL_VERSION-slim
-
-# Install hunspell, Zulip stop words, and run Zulip database
-# init.
-RUN apk add -U --no-cache hunspell-en
-RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix
-COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/local/share/postgresql/tsearch_data/zulip_english.stop
-COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
-COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql

NOTICE

@@ -1,18 +0,0 @@
-Copyright 2012–2015 Dropbox, Inc., 2015–2021 Kandra Labs, Inc., and contributors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this project except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-The software includes some works released by third parties under other
-free and open source licenses. Those works are redistributed under the
-license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.

README.md

@@ -1,83 +1,226 @@
+**[Zulip overview](#zulip-overview)** |
+**[Community](#community)** |
+**[Installing for dev](#installing-the-zulip-development-environment)** |
+**[Installing for production](#running-zulip-in-production)** |
+**[Ways to contribute](#ways-to-contribute)** |
+**[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
+**[License](#license)**
+
 # Zulip overview
 
-[Zulip](https://zulip.com) is an open-source team collaboration tool with unique
-[topic-based threading][why-zulip] that combines the best of email and chat to
-make remote work productive and delightful. Fortune 500 companies, [leading open
-source projects][rust-case-study], and thousands of other organizations use
-Zulip every day. Zulip is the only [modern team chat app][features] that is
-designed for both live and asynchronous conversations.
+Zulip is a powerful, open source group chat application. Written in
+Python and using the Django framework, Zulip supports both private
+messaging and group chats via conversation streams.
 
-Zulip is built by a distributed community of developers from all around the
-world, with 74+ people who have each contributed 100+ commits. With
-over 1000 contributors merging over 500 commits a month, Zulip is the
-largest and fastest growing open source team chat project.
+Zulip also supports fast search, drag-and-drop file uploads, image
+previews, group private messages, audible notifications,
+missed-message emails, desktop apps, and much more.
 
-Come find us on the [development community chat](https://zulip.com/development-community/)!
+Further information on the Zulip project and its features can be found
+at https://www.zulip.org.
 
-[![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
-[![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff)
-[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
-[![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
-[![GitHub Sponsors](https://img.shields.io/github/sponsors/zulip)](https://github.com/sponsors/zulip)
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master)
 
-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
-[why-zulip]: https://zulip.com/why-zulip/
-[rust-case-study]: https://zulip.com/case-studies/rust/
-[features]: https://zulip.com/features/
+## Community
 
-## Getting started
+There are several places online where folks discuss Zulip.
 
-- **Contributing code**. Check out our [guide for new
-  contributors](https://zulip.readthedocs.io/en/latest/contributing/contributing.html)
-  to get started. We have invested in making Zulip’s code highly
-  readable, thoughtfully tested, and easy to modify. Beyond that, we
-  have written an extraordinary 150K words of documentation for Zulip
-  contributors.
+One of those places is our [public Zulip instance](https://zulip.tabbott.net/).
+You can go through the simple signup process at that link, and then you
+will soon be talking to core Zulip developers and other users.  To get
+help in real time, you will have the best luck finding core developers
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get answered
+within a day.
 
-- **Contributing non-code**. [Report an
-  issue](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#reporting-issues),
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip into your language, or [give us
-  feedback](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#user-feedback).
-  We'd love to hear from you, whether you've been using Zulip for years, or are just
-  trying it out for the first time.
+We have a [Google mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
+that is currently pretty low traffic.  It is where we do things like
+announce public meetings or major releases.  You can also use it to
+ask questions about features or possible bugs.
 
-- **Checking Zulip out**. The best way to see Zulip in action is to drop by the
-  [Zulip community server](https://zulip.com/development-community/). We also
-  recommend reading about Zulip's [unique
-  approach](https://zulip.com/why-zulip/) to organizing conversations.
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
+track Zulip-related issues (and store our code, of course).
+Anybody with a Github account should be able to create Issues there
+pertaining to bugs or enhancement requests.  We also use Pull
+Requests as our primary mechanism to receive code contributions.
 
-- **Running a Zulip server**. Self-host Zulip directly on Ubuntu or Debian
-  Linux, in [Docker](https://github.com/zulip/docker-zulip), or with prebuilt
-  images for [Digital Ocean](https://marketplace.digitalocean.com/apps/zulip) and
-  [Render](https://render.com/docs/deploy-zulip).
-  Learn more about [self-hosting Zulip](https://zulip.com/self-hosting/).
+## Installing the Zulip Development environment
 
-- **Using Zulip without setting up a server**. Learn about [Zulip
-  Cloud](https://zulip.com/plans/) hosting options. Zulip sponsors free [Zulip
-  Cloud Standard](https://zulip.com/plans/) for hundreds of worthy
-  organizations, including [fellow open-source
-  projects](https://zulip.com/for/open-source/).
+The Zulip development environment is the recommended option for folks
+interested in trying out Zulip.  This is documented in [the developer
+installation guide][dev-install].
 
-- **Participating in [outreach
-  programs](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#outreach-programs)**
-  like [Google Summer of Code](https://developers.google.com/open-source/gsoc/)
-  and [Outreachy](https://www.outreachy.org/).
+## Running Zulip in production
 
-- **Supporting Zulip**. Advocate for your organization to use Zulip, become a
-  [sponsor](https://github.com/sponsors/zulip), write a review in the mobile app
-  stores, or [help others find
-  Zulip](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#help-others-find-zulip).
+Zulip in production only supports Ubuntu 14.04 right now, but work is
+ongoing on adding support for additional platforms. The installation
+process is documented at https://zulip.org/server.html and in more
+detail in [the
+documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
 
-You may also be interested in reading our [blog](https://blog.zulip.org/), and
-following us on [Twitter](https://twitter.com/zulip) and
-[LinkedIn](https://www.linkedin.com/company/zulip-project/).
+## Ways to contribute
 
-Zulip is distributed under the
-[Apache 2.0](https://github.com/zulip/zulip/blob/main/LICENSE) license.
+Zulip welcomes all forms of contributions!  The page documents the
+Zulip development process.
+
+* **Pull requests**. Before a pull request can be merged, you need to
+to sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
+
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+docs][doc-test]. You can also try out [our new desktop
+client][electron], which is in alpha; we'd appreciate testing and
+[feedback](https://github.com/zulip/zulip-electron/issues/new).
+
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat]. You can also improve
+[Zulip.org][z-org].
+
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list][gg-devel] and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list!  Please report any security issues you discover to
+zulip-security@googlegroups.com.
+
+* **App codebases**. This repository is for the Zulip server and web
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].
+
+* **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
+integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
+and [Trello][]), plus [node.js API bindings][node], and a [full-text search
+PostgreSQL extension][tsearch], as separate repos.
+
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation][transifex] if you're interested in
+contributing!
+
+[cla]: https://opensource.dropbox.com/cla/
+[dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/code-style.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
+[electron]: https://github.com/zulip/zulip-electron
+[gg-devel]: https://groups.google.com/forum/#!forum/zulip-devel
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
+[ios-exp]: https://github.com/zulip/zulip-mobile
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[hubot-adapter]: https://github.com/zulip/hubot-zulip
+[jenkins]: https://github.com/zulip/zulip-jenkins-plugin
+[node]: https://github.com/zulip/zulip-node
+[phab]: https://github.com/zulip/phabricator-to-zulip
+[puppet]: https://github.com/matthewbarr/puppet-zulip
+[redmine]: https://github.com/zulip/zulip-redmine-plugin
+[trello]: https://github.com/zulip/trello-to-zulip
+[tsearch]: https://github.com/zulip/tsearch_extras
+[transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
+[z-org]: https://github.com/zulip/zulip.github.io
+
+## How to get involved with contributing to Zulip
+
+First, subscribe to the Zulip [development discussion mailing
+list][gg-devel].
+
+The Zulip project uses a system of labels in our [issue
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
+
+* [Integrations](https://github.com/zulip/zulip/labels/integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/documentation):
+  The Zulip project loves contributions of new documentation.
+
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html): The
+  projects that are [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).  These are great projects if you're looking to make an impact.
+
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.
+
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
+
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
+
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.
+
+Finally, before implementing a larger feature, we highly recommend
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.
+
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just send an email
+to the Zulip Developers list with your thoughts.
+
+## License
+
+Copyright 2011-2015 Dropbox, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The software includes some works released by third parties under other
+free and open source licenses. Those works are redistributed under the
+license terms under which the works were received. For more details,
+see the ``THIRDPARTY`` file included with this distribution.

SECURITY.md

@@ -1,37 +0,0 @@
-# Security policy
-
-## Reporting a vulnerability
-
-We love responsible reports of (potential) security issues in Zulip,
-whether in the latest release or our development branch.
-
-Our security contact is security@zulip.com. Reporters should expect a
-response within 24 hours.
-
-Please include details on the issue and how you'd like to be credited
-in our release notes when we publish the fix.
-
-Our [security model][security-model] document may be a helpful
-resource.
-
-## Security announcements
-
-We send security announcements to our [announcement mailing
-list](https://groups.google.com/g/zulip-announce). If you are running
-Zulip in production, you should subscribe, by clicking "Join group" at
-the top of that page.
-
-## Supported versions
-
-Zulip provides security support for the latest major release, in the
-form of minor security/maintenance releases.
-
-We work hard to make [upgrades][upgrades] reliable, so that there's no
-reason to run older major releases.
-
-See also our documentation on the [Zulip release
-lifecycle][release-lifecycle].
-
-[security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
-[upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
-[release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html

THIRDPARTY

@@ -0,0 +1,567 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: Zulip
+Upstream-Contact: Zulip Development Discussion <zulip-devel@googlegroups.com>
+Source: https://zulip.org/
+Comment:
+ Unless otherwise noted, the Zulip software is distributed under the Apache
+ License, Version 2.0. The software includes some works released by third
+ parties under other free and open source licenses. Those works are
+ redistributed under the license terms under which the works were received.
+ .
+ While Dropbox has sought to provide complete and accurate licensing
+ information for each FOSS package, Dropbox does not represent or warrant
+ that the licensing information provided herein is correct or error-free.
+ Recipients of the Zulip software should investigate the identified FOSS
+ packages to confirm the accuracy of the licensing information provided.
+ Recipients are also encouraged to notify Dropbox of any inaccurate
+ information or errors found in these notices.
+
+Files: *
+Copyright: 2011-2015 Dropbox, Inc.
+License: Apache-2
+
+Files: api/*
+Copyright: 2012-2014 Dropbox, Inc
+License: Expat
+
+Files: api/integrations/perforce/git_p4.py
+Copyright: 2007 Simon Hausmann <simon@lst.de>,
+ 2007 Trolltech ASA
+License: Expat
+Comment: https://raw.github.com/git/git/34022ba/git-p4.py
+
+Files: bots/jabber_mirror_backend.py
+Copyright: 2013 Permabit, Inc., 2013-2014 Dropbox, Inc.
+License: Expat
+
+Files: confirmation/*
+Copyright: 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
+License: BSD-3-Clause
+
+Files: puppet/apt/*
+Copyright: 2011, Evolving Web Inc.
+License: Expat
+
+Files: puppet/stdlib/*
+Copyright: 2011, Krzysztof Wilczynski
+ 2011, Puppet Labs Inc
+License: Apache-2.0
+
+File: puppet/zulip_internal/files/mediawiki/Auth_remoteuser.php
+Copyright: 2006 Otheus Shelling
+ 2007 Rusty Burchfield
+ 2009 James Kinsman
+ 2010 Daniel Thomas
+ 2010 Ian Ward Comfort
+License: GPL-2.0
+Comment: Not linked.
+
+Files: puppet/zulip/files/nagios_plugins/zulip_base/check_debian_packages
+Copyright: 2005 Francesc Guasch
+License: GPL-2.0
+Comment: Not linked.
+
+Files: puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_postgres.pl
+Copyright: 2007-2015 Greg Sabino Mullane
+License: BSD-2-Clause
+
+Files: puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_website_response.sh
+Copyright: 2011 Chris Freeman
+License: GPL-2.0
+
+Files: puppet/zulip_internal/files/trac/cgi-bin/
+Copyright: 2003-2009 Edgewall Software
+ 2003-2004 Jonas Borgström <jonas@edgewall.com>
+License: BSD-3-Clause
+
+Files: puppet/zulip_internal/files/pagerduty_nagios.pl
+Copyright: 2011, PagerDuty, Inc.
+License: BSD-3-Clause
+
+Files: puppet/zulip_internal/files/zulip-ec2-configure-interfaces
+Copyright: 2013, Dropbox, Inc.
+License: Expat
+
+Files: static/audio/zulip.*
+Copyright: 2011 Vidsyn
+License: CC-0-1.0
+
+Files: static/styles/thirdparty-fonts.css
+Copyright: 2012-2013 Dave Gandy
+License: Expat
+
+Files: static/third/fontawesome/*
+Copyright: 2012-2013 Dave Gandy
+License: SIL-OFL-1.1
+
+Files: static/third/bootstrap/bootstrap-btn.css
+Copyright: 2011-2014 Twitter, Inc
+License: Expat
+
+Files: static/third/bootstrap/css/bootstrap-responsive.css static/third/bootstrap/css/bootstrap.css
+Copyright: 2012 Twitter, Inc
+License: Apache-2.0
+Comment: The software has been modified.
+
+Files: static/third/bootstrap/js/bootstrap.js
+Copyright: 2012 Twitter, Inc
+License: Apache-2.0
+Comment: The software has been modified.
+
+Files: static/third/bootstrap-notify/*
+Copyright: 2013 Nijiko Yonskai
+ 2012 Goodybag, Inc.
+ 2012 Twitter, Inc
+License: Apache-2.0
+Comment: The software has been modified.
+
+Files: static/third/gemoji/images/emoji/unicode/* tools/setup/emoji_dump/*.ttf
+Copyright: Google, Inc.
+License: Apache-2.0
+Comment: These are actually Noto Emoji, not gemoji.
+
+Files: static/third/html5-formdata/formdata.js
+Copyright: 2010 François de Metz
+License: Expat
+Comment: See https://github.com/francois2metz/html5-formdata
+
+Files: src/zulip/static/third/jquery/*
+Copyright: 2011, John Resig
+ 2011, The Dojo Foundation
+License: Expat
+
+Files: static/third/jquery-autosize/jquery.autosize.js
+Copyright: 2013 Jack Moore
+License: Expat
+
+Files: static/third/jquery-caret/*
+Copyright: 2012, 2013 Andrew C. Dvorak
+License: Expat
+
+Files: static/third/jquery-filedrop/jquery.filedrop.js
+Copyright: Resopollution
+License: Expat
+
+Files: static/third/jquery-form/jquery.form.js
+Copyright: M. Alsup
+License: Expat or GPL-2.0
+
+Files: static/third/jquery-idle/jquery.idle.js
+Copyright: 2011-2013 Henrique Boaventura
+License: Expat
+Comment: The software has been modified.
+
+Files: static/third/jquery-mousewheel/jquery.mousewheel.js
+Copyright: 2011 Brandon Aaron
+License: Expat
+
+Files: static/third/jquery-perfect-scrollbar/*
+Copyright: 2012 HyeonJe Jun
+License: Expat
+
+Files: static/third/jquery-throttle-debounce/*
+Copyright: 2010 "Cowboy" Ben Alman
+License: Expat or GPL
+
+Files: static/third/jquery-validate/*
+Copyright: 2006 - 2011 Jörn Zaefferer
+License: Expat
+
+Files: src/zulip/static/third/lazyload/*
+Copyright: 2011 Ryan Grove
+License: Expat
+
+Files: static/third/marked/*
+Copyright: 2011-2013, Christopher Jeffrey
+License: Expat
+
+Files: static/third/string-prototype-codepointat/*
+Copyright: 2014 Mathias Bynens
+License: Expat
+
+Files: static/third/sockjs/sockjs-0.3.4.js
+Copyright: 2011-2012 VMware, Inc.
+ 2012 Douglas Crockford
+License: Expat and public-domain
+
+Files: static/third/sorttable/sorttable.js
+Copyright: 2007 Stuart Langridge
+License: X11
+
+Files: static/third/sourcesans/*
+Copyright: 2010, 2012, 2014 Adobe Systems Incorporated
+License: SIL-OFL-1.1
+
+Files: static/third/spectrum/*
+Copyright: 2013 Brian Grinstead
+License: Expat
+
+Files: static/third/spin/spin.js
+Copyright: 2011-2013 Felix Gnass
+License: Expat
+
+Files: static/third/underscore/underscore.js
+Copyright: 2009-2015 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+License: Expat
+Comment: https://github.com/jashkenas/underscore/blob/master/LICENSE
+
+Files: static/third/winchan/*
+Copyright: 2012 Lloyd Hilaiel
+License: Expat
+Comment: https://github.com/mozilla/winchan
+
+Files: static/third/xdate/*
+Copyright: 2010 C. F., Wong
+License: Expat
+
+Files: static/third/zocial/*
+Copyright: Sam Collins
+License: Expat
+
+Files: tools/inject-messages/othello
+Copyright: Shakespeare
+License: public-domain
+
+Files: tools/jslint/jslint.js
+Copyright: 2002 Douglas Crockford
+License: XXX-good-not-evil
+
+Files: tools/review
+Copyright: 2010 Ksplice, Inc.
+License: Apache-2.0
+
+Files: zerver/lib/bugdown/codehilite.py zerver/lib/bugdown/fenced_code.py
+Copyright: 2006-2008 Waylan Limberg
+License: BSD-3-Clause
+Comment: https://pypi.python.org/pypi/Markdown
+
+Files: zerver/lib/ccache.py
+Copyright: 2013 David Benjamin and Alan Huang
+License: Expat
+
+Files: zerver/lib/decorator.py zerver/management/commands/runtornado.py scripts/setup/generate_secrets.py
+Copyright: Django Software Foundation and individual contributors
+License: BSD-3-Clause
+
+Files: frontend_tests/casperjs/*
+Copyright: 2011-2012 Nicolas Perriault
+ Joyent, Inc. and other Node contributors
+License: Expat
+
+Files: frontend_tests/casperjs/modules/vendors/*
+Copyright: 2011, Jeremy Ashkenas
+License: Expat
+
+
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+     http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the full text of the Apache License version 2 can
+ be found in /usr/share/common-licenses/Apache-2.0.
+
+License: BSD-2-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+    notice(s), this list of conditions and the following disclaimer
+    unmodified other than the allowable addition of one or more
+    copyright notices.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice(s), this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: BSD-3-Clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ .
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ .
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ .
+ 3. Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+License: CC-0-1.0
+ Creative Commons CC0 1.0 Universal
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION
+ ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE
+ USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND
+ DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT
+ OR THE INFORMATION OR WORKS PROVIDED HEREUNDER.
+ .
+ Statement of Purpose
+ .
+ The laws of most jurisdictions throughout the world automatically confer
+ exclusive Copyright and Related Rights (defined below) upon the creator
+ and subsequent owner(s) (each and all, an "owner") of an original work
+ of authorship and/or a database (each, a "Work").
+ .
+ Certain owners wish to permanently relinquish those rights to a Work for
+ the purpose of contributing to a commons of creative, cultural and
+ scientific works ("Commons") that the public can reliably and without
+ fear of later claims of infringement build upon, modify, incorporate in
+ other works, reuse and redistribute as freely as possible in any form
+ whatsoever and for any purposes, including without limitation commercial
+ purposes. These owners may contribute to the Commons to promote the
+ ideal of a free culture and the further production of creative, cultural
+ and scientific works, or to gain reputation or greater distribution for
+ their Work in part through the use and efforts of others.
+ .
+ For these and/or other purposes and motivations, and without any
+ expectation of additional consideration or compensation, the person
+ associating CC0 with a Work (the "Affirmer"), to the extent that he or
+ she is an owner of Copyright and Related Rights in the Work, voluntarily
+ elects to apply CC0 to the Work and publicly distribute the Work under
+ its terms, with knowledge of his or her Copyright and Related Rights in
+ the Work and the meaning and intended legal effect of CC0 on those
+ rights.
+ .
+ 1. Copyright and Related Rights. A Work made available under CC0 may be
+ protected by copyright and related or neighboring rights ("Copyright and
+ Related Rights"). Copyright and Related Rights include, but are not
+ limited to, the following:
+ .
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ .
+ ii. moral rights retained by the original author(s) and/or performer(s);
+ .
+ iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ .
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ .
+ v. rights protecting the extraction, dissemination, use and reuse of
+ data in a Work;
+ .
+ vi. database rights (such as those arising under Directive 96/9/EC of
+ the European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation thereof,
+ including any amended or successor version of such directive); and
+ .
+ vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+ .
+ 2. Waiver. To the greatest extent permitted by, but not in contravention
+ of, applicable law, Affirmer hereby overtly, fully, permanently,
+ irrevocably and unconditionally waives, abandons, and surrenders all of
+ Affirmer's Copyright and Related Rights and associated claims and causes
+ of action, whether now known or unknown (including existing as well as
+ future claims and causes of action), in the Work (i) in all territories
+ worldwide, (ii) for the maximum duration provided by applicable law or
+ treaty (including future time extensions), (iii) in any current or
+ future medium and for any number of copies, and (iv) for any purpose
+ whatsoever, including without limitation commercial, advertising or
+ promotional purposes (the "Waiver"). Affirmer makes the Waiver for the
+ benefit of each member of the public at large and to the detriment of
+ Affirmer's heirs and successors, fully intending that such Waiver shall
+ not be subject to revocation, rescission, cancellation, termination, or
+ any other legal or equitable action to disrupt the quiet enjoyment of
+ the Work by the public as contemplated by Affirmer's express Statement
+ of Purpose.
+ .
+ 3. Public License Fallback. Should any part of the Waiver for any reason
+ be judged legally invalid or ineffective under applicable law, then the
+ Waiver shall be preserved to the maximum extent permitted taking into
+ account Affirmer's express Statement of Purpose. In addition, to the
+ extent the Waiver is so judged Affirmer hereby grants to each affected
+ person a royalty-free, non transferable, non sublicensable, non
+ exclusive, irrevocable and unconditional license to exercise Affirmer's
+ Copyright and Related Rights in the Work (i) in all territories
+ worldwide, (ii) for the maximum duration provided by applicable law or
+ treaty (including future time extensions), (iii) in any current or
+ future medium and for any number of copies, and (iv) for any purpose
+ whatsoever, including without limitation commercial, advertising or
+ promotional purposes (the "License"). The License shall be deemed
+ effective as of the date CC0 was applied by Affirmer to the Work. Should
+ any part of the License for any reason be judged legally invalid or
+ ineffective under applicable law, such partial invalidity or
+ ineffectiveness shall not invalidate the remainder of the License, and
+ in such case Affirmer hereby affirms that he or she will not (i)
+ exercise any of his or her remaining Copyright and Related Rights in the
+ Work or (ii) assert any associated claims and causes of action with
+ respect to the Work, in either case contrary to Affirmer's express
+ Statement of Purpose.
+ .
+ 4. Limitations and Disclaimers.
+ .
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ .
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied, statutory
+ or otherwise, including without limitation warranties of title,
+ merchantability, fitness for a particular purpose, non infringement, or
+ the absence of latent or other defects, accuracy, or the present or
+ absence of errors, whether or not discoverable, all to the greatest
+ extent permissible under applicable law.
+ .
+ c. Affirmer disclaims responsibility for clearing rights of other
+ persons that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the Work.
+ .
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
+
+License: Expat
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+License: GPL-2.0
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2, dated June, 1991.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ can be found in /usr/share/common-licenses/GPL-2 file.
+
+License: SIL-OFL-1.1
+ ---------------------------------------------------------------------------
+ SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+ ---------------------------------------------------------------------------
+ .
+ PREAMBLE
+ .
+ The goals of the Open Font License (OFL) are to stimulate worldwide development
+ of collaborative font projects, to support the font creation efforts of academic
+ and linguistic communities, and to provide a free and open framework in which
+ fonts may be shared and improved in partnership with others.
+ .
+ The OFL allows the licensed fonts to be used, studied, modified and redistributed
+ freely as long as they are not sold by themselves. The fonts, including any
+ derivative works, can be bundled, embedded, redistributed and/or sold with any
+ software provided that any reserved names are not used by derivative works. The
+ fonts and derivatives, however, cannot be released under any other type of license.
+ The requirement for fonts to remain under this license does not apply to any
+ document created using the fonts or their derivatives.
+ .
+ DEFINITIONS
+ .
+ "Font Software" refers to the set of files released by the Copyright Holder(s) under
+ this license and clearly marked as such. This may include source files, build
+ scripts and documentation.
+ .
+ "Reserved Font Name" refers to any names specified as such after the copyright
+ statement(s).
+ .
+ "Original Version" refers to the collection of Font Software components as
+ distributed by the Copyright Holder(s).
+ .
+ "Modified Version" refers to any derivative made by adding to, deleting, or
+ substituting -- in part or in whole -- any of the components of the Original Version,
+ by changing formats or by porting the Font Software to a new environment.
+ .
+ "Author" refers to any designer, engineer, programmer, technical writer or other
+ person who contributed to the Font Software.
+ .
+ PERMISSION & CONDITIONS
+ .
+ Permission is hereby granted, free of charge, to any person obtaining a copy of the
+ Font Software, to use, study, copy, merge, embed, modify, redistribute, and sell
+ modified and unmodified copies of the Font Software, subject to the following
+ conditions:
+ .
+ 1) Neither the Font Software nor any of its individual components, in Original or
+ Modified Versions, may be sold by itself.
+ .
+ 2) Original or Modified Versions of the Font Software may be bundled, redistributed
+ and/or sold with any software, provided that each copy contains the above copyright
+ notice and this license. These can be included either as stand-alone text files,
+ human-readable headers or in the appropriate machine-readable metadata fields within
+ text or binary files as long as those fields can be easily viewed by the user.
+ .
+ 3) No Modified Version of the Font Software may use the Reserved Font Name(s) unless
+ explicit written permission is granted by the corresponding Copyright Holder. This
+ restriction only applies to the primary font name as presented to the users.
+ .
+ 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font Software shall
+ not be used to promote, endorse or advertise any Modified Version, except to
+ acknowledge the contribution(s) of the Copyright Holder(s) and the Author(s) or with
+ their explicit written permission.
+ .
+ 5) The Font Software, modified or unmodified, in part or in whole, must be distributed
+ entirely under this license, and must not be distributed under any other license. The
+ requirement for fonts to remain under this license does not apply to any document
+ created using the Font Software.
+ .
+ TERMINATION
+ .
+ This license becomes null and void if any of the above conditions are not met.
+ .
+ DISCLAIMER
+ .
+ THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+ PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER
+ RIGHT. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR
+ INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE.

Vagrantfile

@@ -1,108 +1,84 @@
 # -*- mode: ruby -*-
 
-Vagrant.require_version ">= 2.2.6"
+VAGRANTFILE_API_VERSION = "2"
+
+def command?(name)
+  `which #{name} > /dev/null 2>&1`
+  $?.success?
+end
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  # For LXC. VirtualBox hosts use a different box, described below.
+  config.vm.box = "fgrehm/trusty64-lxc"
 
-Vagrant.configure("2") do |config|
   # The Zulip development environment runs on 9991 on the guest.
   host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
-  host_ip_addr = "127.0.0.1"
-
-  # System settings for the virtual machine.
-  vm_num_cpus = "2"
-  vm_memory = "2048"
-
-  ubuntu_mirror = ""
-  vboxadd_version = nil
-
-  config.vm.box = "bento/ubuntu-20.04"
+  http_proxy = https_proxy = no_proxy = ""
 
   config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder ".", "/srv/zulip", docker_consistency: "z"
+  config.vm.synced_folder ".", "/srv/zulip"
 
-  vagrant_config_file = ENV["HOME"] + "/.zulip-vagrant-config"
+  vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
   if File.file?(vagrant_config_file)
     IO.foreach(vagrant_config_file) do |line|
       line.chomp!
       key, value = line.split(nil, 2)
       case key
-      when /^([#;]|$)/ # ignore comments
+      when /^([#;]|$)/; # ignore comments
       when "HTTP_PROXY"; http_proxy = value
       when "HTTPS_PROXY"; https_proxy = value
       when "NO_PROXY"; no_proxy = value
       when "HOST_PORT"; host_port = value.to_i
-      when "HOST_IP_ADDR"; host_ip_addr = value
-      when "GUEST_CPUS"; vm_num_cpus = value
-      when "GUEST_MEMORY_MB"; vm_memory = value
-      when "UBUNTU_MIRROR"; ubuntu_mirror = value
-      when "VBOXADD_VERSION"; vboxadd_version = value
       end
     end
   end
 
+  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: "127.0.0.1"
+
   if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
+    if http_proxy != ""
       config.proxy.http = http_proxy
     end
-    if !https_proxy.nil?
+    if https_proxy != ""
       config.proxy.https = https_proxy
     end
-    if !no_proxy.nil?
+    if https_proxy != ""
       config.proxy.no_proxy = no_proxy
     end
-  elsif !http_proxy.nil? or !https_proxy.nil?
-    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
-    # We haven't figured out a workaround.
-    puts "You have specified value for proxy in ~/.zulip-vagrant-config file but did not " \
-         "install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install " \
-         "vagrant-proxyconf` in a terminal.  This error will appear twice."
-    exit
   end
 
-  config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
-  # Specify Docker provider before VirtualBox provider so it's preferred.
-  config.vm.provider "docker" do |d, override|
-    override.vm.box = nil
-    d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
-    d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
-    if !ubuntu_mirror.empty?
-      d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
+  # Specify LXC provider before VirtualBox provider so it's preferred.
+  config.vm.provider "lxc" do |lxc|
+    if command? "lxc-ls"
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
+      if LXC_VERSION >= "1.1.0"
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
+        lxc.customize 'aa_allow_incomplete', 1
+      end
+      if LXC_VERSION >= "2.0.0"
+        lxc.backingstore = 'dir'
+      end
     end
-    d.has_ssh = true
-    d.create_args = ["--ulimit", "nofile=1024:65536"]
   end
 
   config.vm.provider "virtualbox" do |vb, override|
+    override.vm.box = "ubuntu/trusty64"
     # It's possible we can get away with just 1.5GB; more testing needed
-    vb.memory = vm_memory
-    vb.cpus = vm_num_cpus
-
-    if !vboxadd_version.nil?
-      override.vbguest.installer = Class.new(VagrantVbguest::Installers::Ubuntu) do
-        define_method(:host_version) do |reload = false|
-          VagrantVbguest::Version(vboxadd_version)
-        end
-      end
-      override.vbguest.allow_downgrade = true
-      override.vbguest.iso_path = "https://download.virtualbox.org/virtualbox/#{vboxadd_version}/VBoxGuestAdditions_#{vboxadd_version}.iso"
-    end
+    vb.memory = 2048
   end
 
-  config.vm.provider "hyperv" do |h, override|
-    h.memory = vm_memory
-    h.maxmemory = vm_memory
-    h.cpus = vm_num_cpus
-  end
-
-  config.vm.provider "parallels" do |prl, override|
-    prl.memory = vm_memory
-    prl.cpus = vm_num_cpus
-  end
+$provision_script = <<SCRIPT
+set -x
+set -e
+set -o pipefail
+ln -nsf /srv/zulip ~/zulip
+/usr/bin/python /srv/zulip/tools/provision.py | sudo tee -a /var/log/zulip_provision.log
+SCRIPT
 
   config.vm.provision "shell",
-    # We want provision to be run with the permissions of the vagrant user.
+    # We want provision.py to be run with the permissions of the vagrant user.
     privileged: false,
-    path: "tools/setup/vagrant-provision",
-    env: { "UBUNTU_MIRROR" => ubuntu_mirror }
+    inline: $provision_script
 end

analytics/lib/counts.py

@@ -1,821 +0,0 @@
-import logging
-import time
-from collections import OrderedDict, defaultdict
-from datetime import datetime, timedelta
-from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union
-
-from django.conf import settings
-from django.db import connection, models
-from django.db.models import F
-from psycopg2.sql import SQL, Composable, Identifier, Literal
-
-from analytics.models import (
-    BaseCount,
-    FillState,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-    installation_epoch,
-)
-from zerver.lib.logging_util import log_to_file
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
-from zerver.models import Message, Realm, RealmAuditLog, Stream, UserActivityInterval, UserProfile
-
-## Logging setup ##
-
-logger = logging.getLogger("zulip.management")
-log_to_file(logger, settings.ANALYTICS_LOG_PATH)
-
-# You can't subtract timedelta.max from a datetime, so use this instead
-TIMEDELTA_MAX = timedelta(days=365 * 1000)
-
-## Class definitions ##
-
-
-class CountStat:
-    HOUR = "hour"
-    DAY = "day"
-    FREQUENCIES = frozenset([HOUR, DAY])
-
-    @property
-    def time_increment(self) -> timedelta:
-        if self.frequency == CountStat.HOUR:
-            return timedelta(hours=1)
-        return timedelta(days=1)
-
-    def __init__(
-        self,
-        property: str,
-        data_collector: "DataCollector",
-        frequency: str,
-        interval: Optional[timedelta] = None,
-    ) -> None:
-        self.property = property
-        self.data_collector = data_collector
-        # might have to do something different for bitfields
-        if frequency not in self.FREQUENCIES:
-            raise AssertionError(f"Unknown frequency: {frequency}")
-        self.frequency = frequency
-        if interval is not None:
-            self.interval = interval
-        else:
-            self.interval = self.time_increment
-
-    def __repr__(self) -> str:
-        return f"<CountStat: {self.property}>"
-
-    def last_successful_fill(self) -> Optional[datetime]:
-        fillstate = FillState.objects.filter(property=self.property).first()
-        if fillstate is None:
-            return None
-        if fillstate.state == FillState.DONE:
-            return fillstate.end_time
-        return fillstate.end_time - self.time_increment
-
-
-class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
-        CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
-
-
-class DependentCountStat(CountStat):
-    def __init__(
-        self,
-        property: str,
-        data_collector: "DataCollector",
-        frequency: str,
-        interval: Optional[timedelta] = None,
-        dependencies: Sequence[str] = [],
-    ) -> None:
-        CountStat.__init__(self, property, data_collector, frequency, interval=interval)
-        self.dependencies = dependencies
-
-
-class DataCollector:
-    def __init__(
-        self,
-        output_table: Type[BaseCount],
-        pull_function: Optional[Callable[[str, datetime, datetime, Optional[Realm]], int]],
-    ) -> None:
-        self.output_table = output_table
-        self.pull_function = pull_function
-
-
-## CountStat-level operations ##
-
-
-def process_count_stat(
-    stat: CountStat, fill_to_time: datetime, realm: Optional[Realm] = None
-) -> None:
-    # TODO: The realm argument is not yet supported, in that we don't
-    # have a solution for how to update FillState if it is passed.  It
-    # exists solely as partial plumbing for when we do fully implement
-    # doing single-realm analytics runs for use cases like data import.
-    #
-    # Also, note that for the realm argument to be properly supported,
-    # the CountStat object passed in needs to have come from
-    # E.g. get_count_stats(realm), i.e. have the realm_id already
-    # entered into the SQL query defined by the CountState object.
-    verify_UTC(fill_to_time)
-    if floor_to_hour(fill_to_time) != fill_to_time:
-        raise ValueError(f"fill_to_time must be on an hour boundary: {fill_to_time}")
-
-    fill_state = FillState.objects.filter(property=stat.property).first()
-    if fill_state is None:
-        currently_filled = installation_epoch()
-        fill_state = FillState.objects.create(
-            property=stat.property, end_time=currently_filled, state=FillState.DONE
-        )
-        logger.info("INITIALIZED %s %s", stat.property, currently_filled)
-    elif fill_state.state == FillState.STARTED:
-        logger.info("UNDO START %s %s", stat.property, fill_state.end_time)
-        do_delete_counts_at_hour(stat, fill_state.end_time)
-        currently_filled = fill_state.end_time - stat.time_increment
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        logger.info("UNDO DONE %s", stat.property)
-    elif fill_state.state == FillState.DONE:
-        currently_filled = fill_state.end_time
-    else:
-        raise AssertionError(f"Unknown value for FillState.state: {fill_state.state}.")
-
-    if isinstance(stat, DependentCountStat):
-        for dependency in stat.dependencies:
-            dependency_fill_time = COUNT_STATS[dependency].last_successful_fill()
-            if dependency_fill_time is None:
-                logger.warning(
-                    "DependentCountStat %s run before dependency %s.", stat.property, dependency
-                )
-                return
-            fill_to_time = min(fill_to_time, dependency_fill_time)
-
-    currently_filled = currently_filled + stat.time_increment
-    while currently_filled <= fill_to_time:
-        logger.info("START %s %s", stat.property, currently_filled)
-        start = time.time()
-        do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
-        do_fill_count_stat_at_hour(stat, currently_filled, realm)
-        do_update_fill_state(fill_state, currently_filled, FillState.DONE)
-        end = time.time()
-        currently_filled = currently_filled + stat.time_increment
-        logger.info("DONE %s (%dms)", stat.property, (end - start) * 1000)
-
-
-def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
-    fill_state.end_time = end_time
-    fill_state.state = state
-    fill_state.save()
-
-
-# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is time-zone-aware. It is the caller's responsibility to enforce this!
-def do_fill_count_stat_at_hour(
-    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
-) -> None:
-    start_time = end_time - stat.interval
-    if not isinstance(stat, LoggingCountStat):
-        timer = time.time()
-        assert stat.data_collector.pull_function is not None
-        rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time, realm)
-        logger.info(
-            "%s run pull_function (%dms/%sr)",
-            stat.property,
-            (time.time() - timer) * 1000,
-            rows_added,
-        )
-    do_aggregate_to_summary_table(stat, end_time, realm)
-
-
-def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
-    if isinstance(stat, LoggingCountStat):
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.output_table in [UserCount, StreamCount]:
-            RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-    else:
-        UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-
-
-def do_aggregate_to_summary_table(
-    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
-) -> None:
-    cursor = connection.cursor()
-
-    # Aggregate into RealmCount
-    output_table = stat.data_collector.output_table
-    if realm is not None:
-        realm_clause: Composable = SQL("AND zerver_realm.id = {}").format(Literal(realm.id))
-    else:
-        realm_clause = SQL("")
-
-    if output_table in (UserCount, StreamCount):
-        realmcount_query = SQL(
-            """
-            INSERT INTO analytics_realmcount
-                (realm_id, value, property, subgroup, end_time)
-            SELECT
-                zerver_realm.id, COALESCE(sum({output_table}.value), 0), %(property)s,
-                {output_table}.subgroup, %(end_time)s
-            FROM zerver_realm
-            JOIN {output_table}
-            ON
-                zerver_realm.id = {output_table}.realm_id
-            WHERE
-                {output_table}.property = %(property)s AND
-                {output_table}.end_time = %(end_time)s
-                {realm_clause}
-            GROUP BY zerver_realm.id, {output_table}.subgroup
-        """
-        ).format(
-            output_table=Identifier(output_table._meta.db_table),
-            realm_clause=realm_clause,
-        )
-        start = time.time()
-        cursor.execute(
-            realmcount_query,
-            {
-                "property": stat.property,
-                "end_time": end_time,
-            },
-        )
-        end = time.time()
-        logger.info(
-            "%s RealmCount aggregation (%dms/%sr)",
-            stat.property,
-            (end - start) * 1000,
-            cursor.rowcount,
-        )
-
-    if realm is None:
-        # Aggregate into InstallationCount.  Only run if we just
-        # processed counts for all realms.
-        #
-        # TODO: Add support for updating installation data after
-        # changing an individual realm's values.
-        installationcount_query = SQL(
-            """
-            INSERT INTO analytics_installationcount
-                (value, property, subgroup, end_time)
-            SELECT
-                sum(value), %(property)s, analytics_realmcount.subgroup, %(end_time)s
-            FROM analytics_realmcount
-            WHERE
-                property = %(property)s AND
-                end_time = %(end_time)s
-            GROUP BY analytics_realmcount.subgroup
-        """
-        )
-        start = time.time()
-        cursor.execute(
-            installationcount_query,
-            {
-                "property": stat.property,
-                "end_time": end_time,
-            },
-        )
-        end = time.time()
-        logger.info(
-            "%s InstallationCount aggregation (%dms/%sr)",
-            stat.property,
-            (end - start) * 1000,
-            cursor.rowcount,
-        )
-
-    cursor.close()
-
-
-## Utility functions called from outside counts.py ##
-
-
-# called from zerver.actions; should not throw any errors
-def do_increment_logging_stat(
-    zerver_object: Union[Realm, UserProfile, Stream],
-    stat: CountStat,
-    subgroup: Optional[Union[str, int, bool]],
-    event_time: datetime,
-    increment: int = 1,
-) -> None:
-    if not increment:
-        return
-
-    table = stat.data_collector.output_table
-    if table == RealmCount:
-        assert isinstance(zerver_object, Realm)
-        id_args: Dict[str, Union[Realm, UserProfile, Stream]] = {"realm": zerver_object}
-    elif table == UserCount:
-        assert isinstance(zerver_object, UserProfile)
-        id_args = {"realm": zerver_object.realm, "user": zerver_object}
-    else:  # StreamCount
-        assert isinstance(zerver_object, Stream)
-        id_args = {"realm": zerver_object.realm, "stream": zerver_object}
-
-    if stat.frequency == CountStat.DAY:
-        end_time = ceiling_to_day(event_time)
-    else:  # CountStat.HOUR:
-        end_time = ceiling_to_hour(event_time)
-
-    row, created = table.objects.get_or_create(
-        property=stat.property,
-        subgroup=subgroup,
-        end_time=end_time,
-        defaults={"value": increment},
-        **id_args,
-    )
-    if not created:
-        row.value = F("value") + increment
-        row.save(update_fields=["value"])
-
-
-def do_drop_all_analytics_tables() -> None:
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-
-def do_drop_single_stat(property: str) -> None:
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-
-## DataCollector-level operations ##
-
-QueryFn = Callable[[Dict[str, Composable]], Composable]
-
-
-def do_pull_by_sql_query(
-    property: str,
-    start_time: datetime,
-    end_time: datetime,
-    query: QueryFn,
-    group_by: Optional[Tuple[Type[models.Model], str]],
-) -> int:
-    if group_by is None:
-        subgroup: Composable = SQL("NULL")
-        group_by_clause: Composable = SQL("")
-    else:
-        subgroup = Identifier(group_by[0]._meta.db_table, group_by[1])
-        group_by_clause = SQL(", {}").format(subgroup)
-
-    # We do string replacement here because cursor.execute will reject a
-    # group_by_clause given as a param.
-    # We pass in the datetimes as params to cursor.execute so that we don't have to
-    # think about how to convert python datetimes to SQL datetimes.
-    query_ = query(
-        {
-            "subgroup": subgroup,
-            "group_by_clause": group_by_clause,
-        }
-    )
-    cursor = connection.cursor()
-    cursor.execute(
-        query_,
-        {
-            "property": property,
-            "time_start": start_time,
-            "time_end": end_time,
-        },
-    )
-    rowcount = cursor.rowcount
-    cursor.close()
-    return rowcount
-
-
-def sql_data_collector(
-    output_table: Type[BaseCount],
-    query: QueryFn,
-    group_by: Optional[Tuple[Type[models.Model], str]],
-) -> DataCollector:
-    def pull_function(
-        property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
-    ) -> int:
-        # The pull function type needs to accept a Realm argument
-        # because the 'minutes_active::day' CountStat uses
-        # DataCollector directly for do_pull_minutes_active, which
-        # requires the realm argument.  We ignore it here, because the
-        # realm should have been already encoded in the `query` we're
-        # passed.
-        return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
-
-    return DataCollector(output_table, pull_function)
-
-
-def do_pull_minutes_active(
-    property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
-) -> int:
-    user_activity_intervals = (
-        UserActivityInterval.objects.filter(
-            end__gt=start_time,
-            start__lt=end_time,
-        )
-        .select_related(
-            "user_profile",
-        )
-        .values_list("user_profile_id", "user_profile__realm_id", "start", "end")
-    )
-
-    seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
-    for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
-        if realm is None or realm.id == realm_id:
-            start = max(start_time, interval_start)
-            end = min(end_time, interval_end)
-            seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
-
-    rows = [
-        UserCount(
-            user_id=ids[0],
-            realm_id=ids[1],
-            property=property,
-            end_time=end_time,
-            value=int(seconds // 60),
-        )
-        for ids, seconds in seconds_active.items()
-        if seconds >= 60
-    ]
-    UserCount.objects.bulk_create(rows)
-    return len(rows)
-
-
-def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
-        %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_message
-    ON
-        zerver_userprofile.id = zerver_message.sender_id
-    WHERE
-        zerver_userprofile.date_joined < %(time_end)s AND
-        zerver_message.date_sent >= %(time_start)s AND
-        {realm_clause}
-        zerver_message.date_sent < %(time_end)s
-    GROUP BY zerver_userprofile.id {group_by_clause}
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-# Note: ignores the group_by / group_by_clause.
-def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_usercount
-            (realm_id, user_id, value, property, subgroup, end_time)
-    SELECT realm_id, id, SUM(count) AS value, %(property)s, message_type, %(time_end)s
-    FROM
-    (
-        SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
-        CASE WHEN
-                  zerver_recipient.type = 1 THEN 'private_message'
-             WHEN
-                  zerver_recipient.type = 3 THEN 'huddle_message'
-             WHEN
-                  zerver_stream.invite_only = TRUE THEN 'private_stream'
-             ELSE 'public_stream'
-        END
-        message_type
-
-        FROM zerver_userprofile
-        JOIN zerver_message
-        ON
-            zerver_userprofile.id = zerver_message.sender_id AND
-            zerver_message.date_sent >= %(time_start)s AND
-            {realm_clause}
-            zerver_message.date_sent < %(time_end)s
-        JOIN zerver_recipient
-        ON
-            zerver_message.recipient_id = zerver_recipient.id
-        LEFT JOIN zerver_stream
-        ON
-            zerver_recipient.type_id = zerver_stream.id
-        GROUP BY
-            zerver_userprofile.realm_id, zerver_userprofile.id,
-            zerver_recipient.type, zerver_stream.invite_only
-    ) AS subquery
-    GROUP BY realm_id, id, message_type
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-# This query joins to the UserProfile table since all current queries that
-# use this also subgroup on UserProfile.is_bot. If in the future there is a
-# stat that counts messages by stream and doesn't need the UserProfile
-# table, consider writing a new query for efficiency.
-def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_streamcount
-        (stream_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_stream.id, zerver_stream.realm_id, count(*), %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_stream
-    JOIN zerver_recipient
-    ON
-        zerver_stream.id = zerver_recipient.type_id
-    JOIN zerver_message
-    ON
-        zerver_recipient.id = zerver_message.recipient_id
-    JOIN zerver_userprofile
-    ON
-        zerver_message.sender_id = zerver_userprofile.id
-    WHERE
-        zerver_stream.date_created < %(time_end)s AND
-        zerver_recipient.type = 2 AND
-        zerver_message.date_sent >= %(time_start)s AND
-        {realm_clause}
-        zerver_message.date_sent < %(time_end)s
-    GROUP BY zerver_stream.id {group_by_clause}
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-# Hardcodes the query needed by active_users:is_bot:day, since that is
-# currently the only stat that uses this.
-def count_user_by_realm_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_realm
-    JOIN zerver_userprofile
-    ON
-        zerver_realm.id = zerver_userprofile.realm_id
-    WHERE
-        zerver_realm.date_created < %(time_end)s AND
-        zerver_userprofile.date_joined >= %(time_start)s AND
-        zerver_userprofile.date_joined < %(time_end)s AND
-        {realm_clause}
-        zerver_userprofile.is_active = TRUE
-    GROUP BY zerver_realm.id {group_by_clause}
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-# Currently hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
-# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
-# In particular, it's important to ensure that migrations don't cause that to happen.
-def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        ral1.modified_user_id, ral1.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_realmauditlog ral1
-    JOIN (
-        SELECT modified_user_id, max(event_time) AS max_event_time
-        FROM zerver_realmauditlog
-        WHERE
-            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
-            {realm_clause}
-            event_time < %(time_end)s
-        GROUP BY modified_user_id
-    ) ral2
-    ON
-        ral1.event_time = max_event_time AND
-        ral1.modified_user_id = ral2.modified_user_id
-    JOIN zerver_userprofile
-    ON
-        ral1.modified_user_id = zerver_userprofile.id
-    WHERE
-        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
-    """
-    ).format(
-        **kwargs,
-        user_created=Literal(RealmAuditLog.USER_CREATED),
-        user_activated=Literal(RealmAuditLog.USER_ACTIVATED),
-        user_deactivated=Literal(RealmAuditLog.USER_DEACTIVATED),
-        user_reactivated=Literal(RealmAuditLog.USER_REACTIVATED),
-        realm_clause=realm_clause,
-    )
-
-
-def check_useractivityinterval_by_user_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_usercount
-        (user_id, realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_userprofile.id, zerver_userprofile.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_userprofile
-    JOIN zerver_useractivityinterval
-    ON
-        zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
-    WHERE
-        zerver_useractivityinterval.end >= %(time_start)s AND
-        {realm_clause}
-        zerver_useractivityinterval.start < %(time_end)s
-    GROUP BY zerver_userprofile.id {group_by_clause}
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-def count_realm_active_humans_query(realm: Optional[Realm]) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
-    return lambda kwargs: SQL(
-        """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        usercount1.realm_id, count(*), %(property)s, NULL, %(time_end)s
-    FROM (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = 'active_users_audit:is_bot:day' AND
-            subgroup = 'false' AND
-            {realm_clause}
-            end_time = %(time_end)s
-    ) usercount1
-    JOIN (
-        SELECT realm_id, user_id
-        FROM analytics_usercount
-        WHERE
-            property = '15day_actives::day' AND
-            {realm_clause}
-            end_time = %(time_end)s
-    ) usercount2
-    ON
-        usercount1.user_id = usercount2.user_id
-    GROUP BY usercount1.realm_id
-"""
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
-# Currently unused and untested
-count_stream_by_realm_query = lambda kwargs: SQL(
-    """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
-    SELECT
-        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_realm
-    JOIN zerver_stream
-    ON
-        zerver_realm.id = zerver_stream.realm_id AND
-    WHERE
-        zerver_realm.date_created < %(time_end)s AND
-        zerver_stream.date_created >= %(time_start)s AND
-        zerver_stream.date_created < %(time_end)s
-    GROUP BY zerver_realm.id {group_by_clause}
-"""
-).format(**kwargs)
-
-
-def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
-    ## CountStat declarations ##
-
-    count_stats_ = [
-        # Messages sent stats
-        # Stats that count the number of messages sent in various ways.
-        # These are also the set of stats that read from the Message table.
-        CountStat(
-            "messages_sent:is_bot:hour",
-            sql_data_collector(
-                UserCount, count_message_by_user_query(realm), (UserProfile, "is_bot")
-            ),
-            CountStat.HOUR,
-        ),
-        CountStat(
-            "messages_sent:message_type:day",
-            sql_data_collector(UserCount, count_message_type_by_user_query(realm), None),
-            CountStat.DAY,
-        ),
-        CountStat(
-            "messages_sent:client:day",
-            sql_data_collector(
-                UserCount, count_message_by_user_query(realm), (Message, "sending_client_id")
-            ),
-            CountStat.DAY,
-        ),
-        CountStat(
-            "messages_in_stream:is_bot:day",
-            sql_data_collector(
-                StreamCount, count_message_by_stream_query(realm), (UserProfile, "is_bot")
-            ),
-            CountStat.DAY,
-        ),
-        # Number of users stats
-        # Stats that count the number of active users in the UserProfile.is_active sense.
-        # 'active_users_audit:is_bot:day' is the canonical record of which users were
-        # active on which days (in the UserProfile.is_active sense).
-        # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
-        CountStat(
-            "active_users_audit:is_bot:day",
-            sql_data_collector(
-                UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
-            ),
-            CountStat.DAY,
-        ),
-        # Important note: LoggingCountStat objects aren't passed the
-        # Realm argument, because by nature they have a logging
-        # structure, not a pull-from-database structure, so there's no
-        # way to compute them for a single realm after the fact (the
-        # use case for passing a Realm argument).
-        # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
-        # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
-        # sum sequence of 'active_users_log:is_bot:day', for any realm that
-        # started after the latter stat was introduced.
-        LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
-        # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
-        # approximation, e.g. if a user is deactivated between the end of the
-        # day and when this stat is run, they won't be counted. However, is the
-        # simplest of the three to inspect by hand.
-        CountStat(
-            "active_users:is_bot:day",
-            sql_data_collector(
-                RealmCount, count_user_by_realm_query(realm), (UserProfile, "is_bot")
-            ),
-            CountStat.DAY,
-            interval=TIMEDELTA_MAX,
-        ),
-        # Messages read stats.  messages_read::hour is the total
-        # number of messages read, whereas
-        # messages_read_interactions::hour tries to count the total
-        # number of UI interactions resulting in messages being marked
-        # as read (imperfect because of batching of some request
-        # types, but less likely to be overwhelmed by a single bulk
-        # operation).
-        LoggingCountStat("messages_read::hour", UserCount, CountStat.HOUR),
-        LoggingCountStat("messages_read_interactions::hour", UserCount, CountStat.HOUR),
-        # User activity stats
-        # Stats that measure user activity in the UserActivityInterval sense.
-        CountStat(
-            "1day_actives::day",
-            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
-            CountStat.DAY,
-            interval=timedelta(days=1) - UserActivityInterval.MIN_INTERVAL_LENGTH,
-        ),
-        CountStat(
-            "7day_actives::day",
-            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
-            CountStat.DAY,
-            interval=timedelta(days=7) - UserActivityInterval.MIN_INTERVAL_LENGTH,
-        ),
-        CountStat(
-            "15day_actives::day",
-            sql_data_collector(UserCount, check_useractivityinterval_by_user_query(realm), None),
-            CountStat.DAY,
-            interval=timedelta(days=15) - UserActivityInterval.MIN_INTERVAL_LENGTH,
-        ),
-        CountStat(
-            "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
-        ),
-        # Rate limiting stats
-        # Used to limit the number of invitation emails sent by a realm
-        LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
-        # Dependent stats
-        # Must come after their dependencies.
-        # Canonical account of the number of active humans in a realm on each day.
-        DependentCountStat(
-            "realm_active_humans::day",
-            sql_data_collector(RealmCount, count_realm_active_humans_query(realm), None),
-            CountStat.DAY,
-            dependencies=["active_users_audit:is_bot:day", "15day_actives::day"],
-        ),
-    ]
-
-    return OrderedDict((stat.property, stat) for stat in count_stats_)
-
-
-# To avoid refactoring for now COUNT_STATS can be used as before
-COUNT_STATS = get_count_stats()

analytics/lib/fixtures.py

@@ -1,79 +0,0 @@
-from math import sqrt
-from random import gauss, random, seed
-from typing import List
-
-from analytics.lib.counts import CountStat
-
-
-def generate_time_series_data(
-    days: int = 100,
-    business_hours_base: float = 10,
-    non_business_hours_base: float = 10,
-    growth: float = 1,
-    autocorrelation: float = 0,
-    spikiness: float = 1,
-    holiday_rate: float = 0,
-    frequency: str = CountStat.DAY,
-    partial_sum: bool = False,
-    random_seed: int = 26,
-) -> List[int]:
-    """
-    Generate semi-realistic looking time series data for testing analytics graphs.
-
-    days -- Number of days of data. Is the number of data points generated if
-        frequency is CountStat.DAY.
-    business_hours_base -- Average value during a business hour (or day) at beginning of
-        time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
-    non_business_hours_base -- The above, for non-business hours/days.
-    growth -- Ratio between average values at end of time series and beginning of time series.
-    autocorrelation -- Makes neighboring data points look more like each other. At 0 each
-        point is unaffected by the previous point, and at 1 each point is a deterministic
-        function of the previous point.
-    spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
-        the variance.
-    holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
-    frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
-    random_seed -- Seed for random number generator.
-    """
-    if frequency == CountStat.HOUR:
-        length = days * 24
-        seasonality = [non_business_hours_base] * 24 * 7
-        for day in range(5):
-            for hour in range(8):
-                seasonality[24 * day + hour] = business_hours_base
-        holidays = []
-        for i in range(days):
-            holidays.extend([random() < holiday_rate] * 24)
-    elif frequency == CountStat.DAY:
-        length = days
-        seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
-            24 * non_business_hours_base
-        ] * 2
-        holidays = [random() < holiday_rate for i in range(days)]
-    else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
-    if length < 2:
-        raise AssertionError(
-            f"Must be generating at least 2 data points. Currently generating {length}"
-        )
-    growth_base = growth ** (1.0 / (length - 1))
-    values_no_noise = [
-        seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)
-    ]
-
-    seed(random_seed)
-    noise_scalars = [gauss(0, 1)]
-    for i in range(1, length):
-        noise_scalars.append(
-            noise_scalars[-1] * autocorrelation + gauss(0, 1) * (1 - autocorrelation)
-        )
-
-    values = [
-        0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
-        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)
-    ]
-    if partial_sum:
-        for i in range(1, length):
-            values[i] = values[i - 1] + values[i]
-    return [max(v, 0) for v in values]

analytics/lib/time_utils.py

@@ -1,33 +0,0 @@
-from datetime import datetime, timedelta
-from typing import List, Optional
-
-from analytics.lib.counts import CountStat
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
-
-
-# If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
-# If min_length is greater than 0, pads the list to the left.
-# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
-# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(
-    start: datetime, end: datetime, frequency: str, min_length: Optional[int]
-) -> List[datetime]:
-    verify_UTC(start)
-    verify_UTC(end)
-    if frequency == CountStat.HOUR:
-        end = floor_to_hour(end)
-        step = timedelta(hours=1)
-    elif frequency == CountStat.DAY:
-        end = floor_to_day(end)
-        step = timedelta(days=1)
-    else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
-
-    times = []
-    if min_length is not None:
-        start = min(start, end - (min_length - 1) * step)
-    current = end
-    while current >= start:
-        times.append(current)
-        current -= step
-    return list(reversed(times))

analytics/management/commands/active_user_stats.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from typing import Any
+
+from zerver.models import UserPresence, UserActivity
+from zerver.lib.utils import statsd, statsd_key
+
+from datetime import datetime, timedelta
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = """Sends active user statistics to statsd.
+
+    Run as a cron job that runs every 10 minutes."""
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        # Get list of all active users in the last 1 week
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+
+        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
+
+        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
+        hour_buckets = [0.16, 2, 12, 24, 48, 168]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+
+        for last_presence in users:
+            if last_presence.status == UserPresence.IDLE:
+                known_active = last_presence.timestamp - timedelta(minutes=30)
+            else:
+                known_active = last_presence.timestamp
+
+            for bucket in hour_buckets:
+                if bucket not in user_info[last_presence.user_profile.realm.domain]:
+                    user_info[last_presence.user_profile.realm.domain][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
+
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
+
+        # Also do stats for how many users have been reading the app.
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
+        user_info = defaultdict(dict)
+        for activity in users_reading:
+            for bucket in hour_buckets:
+                if bucket not in user_info[activity.user_profile.realm.domain]:
+                    user_info[activity.user_profile.realm.domain][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers reading for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))

analytics/management/commands/active_user_stats_by_day.py

@@ -0,0 +1,27 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import datetime
+import pytz
+
+from optparse import make_option
+from typing import Any
+from django.core.management.base import BaseCommand
+from zerver.lib.statistics import activity_averages_during_day
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity for a given day."
+
+    option_list = BaseCommand.option_list + \
+        (make_option('--date', default=None, action='store',
+                     help="Day to query in format 2013-12-05.  Default is yesterday"),)
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options["date"] is None:
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")

analytics/management/commands/analyze_mit.py

@@ -0,0 +1,86 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from optparse import make_option
+from django.core.management.base import BaseCommand
+from zerver.models import Recipient, Message
+from zerver.lib.timestamp import timestamp_to_datetime
+import datetime
+import time
+import logging
+
+def compute_stats(log_level):
+    # type: (int) -> None
+    logger = logging.getLogger()
+    logger.setLevel(log_level)
+
+    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
+    mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
+                                       recipient__type=Recipient.STREAM,
+                                       pub_date__gt=one_week_ago)
+    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
+        mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
+    # Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
+    mit_query = mit_query.exclude(sender__email__contains=("/"))
+    mit_query = mit_query.exclude(sender__email__contains=("aim.com"))
+    mit_query = mit_query.exclude(
+        sender__email__in=["rss@mit.edu", "bash@mit.edu", "apache@mit.edu",
+                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
+                           "root@mit.edu", "nagios@mit.edu",
+                           "www-data|local-realm@mit.edu"])
+    user_counts = {} # type: Dict[str, Dict[str, int]]
+    for m in mit_query.select_related("sending_client", "sender"):
+        email = m.sender.email
+        user_counts.setdefault(email, {})
+        user_counts[email].setdefault(m.sending_client.name, 0)
+        user_counts[email][m.sending_client.name] += 1
+
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
+    for email, counts in user_counts.items():
+        total_user_counts.setdefault(email, 0)
+        for client_name, count in counts.items():
+            total_counts.setdefault(client_name, 0)
+            total_counts[client_name] += count
+            total_user_counts[email] += count
+
+    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
+    top_percents = {} # type: Dict[int, float]
+    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
+        top_percents[size] = 0.0
+    for i, email in enumerate(sorted(total_user_counts.keys(),
+                                     key=lambda x: -total_user_counts[x])):
+        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
+                               total_user_counts[email], 1)
+        for size in top_percents.keys():
+            top_percents.setdefault(size, 0)
+            if i < size:
+                top_percents[size] += (percent_zulip * 1.0 / size)
+
+        logging.debug("%40s | %10s | %s%%" % (email, total_user_counts[email],
+                                              percent_zulip))
+
+    logging.info("")
+    for size in sorted(top_percents.keys()):
+        logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
+
+    grand_total = sum(total_counts.values())
+    print(grand_total)
+    logging.info("%15s | %s" % ("Client", "Percentage"))
+    for client in total_counts.keys():
+        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + \
+        (make_option('--verbose', default=False, action='store_true'),)
+
+    help = "Compute statistics on MIT Zephyr usage."
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        level = logging.INFO
+        if options["verbose"]:
+            level = logging.DEBUG
+        compute_stats(level)

analytics/management/commands/analyze_user_activity.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any, Dict
+
+from zerver.lib.statistics import seconds_usage_between
+
+from optparse import make_option
+from django.core.management.base import BaseCommand
+from zerver.models import UserProfile
+import datetime
+from django.utils.timezone import utc
+
+def analyze_activity(options):
+    # type: (Dict[str, Any]) -> None
+    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
+    day_end = day_start + datetime.timedelta(days=options["duration"])
+
+    user_profile_query = UserProfile.objects.all()
+    if options["realm"]:
+        user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
+
+    print("Per-user online duration:\n")
+    total_duration = datetime.timedelta(0)
+    for user_profile in user_profile_query:
+        duration = seconds_usage_between(user_profile, day_start, day_end)
+
+        if duration == datetime.timedelta(0):
+            continue
+
+        total_duration += duration
+        print("%-*s%s" % (37, user_profile.email, duration,))
+
+    print("\nTotal Duration:                      %s" % (total_duration,))
+    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
+    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
+
+class Command(BaseCommand):
+    help = """Report analytics of user activity on a per-user and realm basis.
+
+This command aggregates user activity data that is collected by each user using Zulip. It attempts
+to approximate how much each user has been using Zulip per day, measured by recording each 15 minute
+period where some activity has occurred (mouse move or keyboard activity).
+
+It will correctly not count server-initiated reloads in the activity statistics.
+
+The duration flag can be used to control how many days to show usage duration for
+
+Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
+
+By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
+is shown for all realms"""
+
+    option_list = BaseCommand.option_list + (
+        make_option('--realm', action='store'),
+        make_option('--date', action='store', default="2013-09-06"),
+        make_option('--duration', action='store', default=1, type=int,
+                    help="How many days to show usage information for"),
+        )
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        analyze_activity(options)

analytics/management/commands/check_analytics_state.py

@@ -1,90 +0,0 @@
-import os
-import time
-from datetime import timedelta
-from typing import Any, Dict
-
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.models import installation_epoch
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
-from zerver.models import Realm
-
-states = {
-    0: "OK",
-    1: "WARNING",
-    2: "CRITICAL",
-    3: "UNKNOWN",
-}
-
-
-class Command(BaseCommand):
-    help = """Checks FillState table.
-
-    Run as a cron job that runs every hour."""
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        fill_state = self.get_fill_state()
-        status = fill_state["status"]
-        message = fill_state["message"]
-
-        state_file_path = "/var/lib/nagios_state/check-analytics-state"
-        state_file_tmp = state_file_path + "-tmp"
-
-        with open(state_file_tmp, "w") as f:
-            f.write(f"{int(time.time())}|{status}|{states[status]}|{message}\n")
-        os.rename(state_file_tmp, state_file_path)
-
-    def get_fill_state(self) -> Dict[str, Any]:
-        if not Realm.objects.exists():
-            return {"status": 0, "message": "No realms exist, so not checking FillState."}
-
-        warning_unfilled_properties = []
-        critical_unfilled_properties = []
-        for property, stat in COUNT_STATS.items():
-            last_fill = stat.last_successful_fill()
-            if last_fill is None:
-                last_fill = installation_epoch()
-            try:
-                verify_UTC(last_fill)
-            except TimeZoneNotUTCError:
-                return {"status": 2, "message": f"FillState not in UTC for {property}"}
-
-            if stat.frequency == CountStat.DAY:
-                floor_function = floor_to_day
-                warning_threshold = timedelta(hours=26)
-                critical_threshold = timedelta(hours=50)
-            else:  # CountStat.HOUR
-                floor_function = floor_to_hour
-                warning_threshold = timedelta(minutes=90)
-                critical_threshold = timedelta(minutes=150)
-
-            if floor_function(last_fill) != last_fill:
-                return {
-                    "status": 2,
-                    "message": f"FillState not on {stat.frequency} boundary for {property}",
-                }
-
-            time_to_last_fill = timezone_now() - last_fill
-            if time_to_last_fill > critical_threshold:
-                critical_unfilled_properties.append(property)
-            elif time_to_last_fill > warning_threshold:
-                warning_unfilled_properties.append(property)
-
-        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return {"status": 0, "message": "FillState looks fine."}
-        if len(critical_unfilled_properties) == 0:
-            return {
-                "status": 1,
-                "message": "Missed filling {} once.".format(
-                    ", ".join(warning_unfilled_properties),
-                ),
-            }
-        return {
-            "status": 2,
-            "message": "Missed filling {} once. Missed filling {} at least twice.".format(
-                ", ".join(warning_unfilled_properties),
-                ", ".join(critical_unfilled_properties),
-            ),
-        }

analytics/management/commands/clear_analytics_tables.py

@@ -1,21 +0,0 @@
-from argparse import ArgumentParser
-from typing import Any
-
-from django.core.management.base import BaseCommand, CommandError
-
-from analytics.lib.counts import do_drop_all_analytics_tables
-
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        if options["force"]:
-            do_drop_all_analytics_tables()
-        else:
-            raise CommandError(
-                "Would delete all data from analytics tables (!); use --force to do so."
-            )

analytics/management/commands/clear_single_stat.py

@@ -1,23 +0,0 @@
-from argparse import ArgumentParser
-from typing import Any
-
-from django.core.management.base import BaseCommand, CommandError
-
-from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
-
-
-class Command(BaseCommand):
-    help = """Clear analytics tables."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument("--force", action="store_true", help="Actually do it.")
-        parser.add_argument("--property", help="The property of the stat to be cleared.")
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        property = options["property"]
-        if property not in COUNT_STATS:
-            raise CommandError(f"Invalid property: {property}")
-        if not options["force"]:
-            raise CommandError("No action taken. Use --force.")
-
-        do_drop_single_stat(property)

analytics/management/commands/client_activity.py

@@ -0,0 +1,81 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
+from django.db.models import Count, QuerySet
+
+from zerver.models import UserActivity, UserProfile, Realm, \
+    get_realm, get_user_profile_by_email
+
+import datetime
+
+class Command(BaseCommand):
+    help = """Report rough client activity globally, for a realm, or for a user
+
+Usage examples:
+
+python manage.py client_activity
+python manage.py client_activity zulip.com
+python manage.py client_activity jesstess@zulip.com"""
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
+                            help="realm or user to estimate client activity for")
+
+    def compute_activity(self, user_activity_objects):
+        # type: (QuerySet) -> None
+        # Report data from the past week.
+        #
+        # This is a rough report of client activity because we inconsistently
+        # register activity from various clients; think of it as telling you
+        # approximately how many people from a group have used a particular
+        # client recently. For example, this might be useful to get a sense of
+        # how popular different versions of a desktop client are.
+        #
+        # Importantly, this does NOT tell you anything about the relative
+        # volumes of requests from clients.
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
+        client_counts = user_activity_objects.filter(
+            last_visit__gt=threshold).values("client__name").annotate(
+            count=Count('client__name'))
+
+        total = 0
+        counts = []
+        for client_type in client_counts:
+            count = client_type["count"]
+            client = client_type["client__name"]
+            total += count
+            counts.append((count, client))
+
+        counts.sort()
+
+        for count in counts:
+            print("%25s %15d" % (count[1], count[0]))
+        print("Total:", total)
+
+
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['arg'] is None:
+            # Report global activity.
+            self.compute_activity(UserActivity.objects.all())
+        else:
+            arg = options['arg']
+            try:
+                # Report activity for a user.
+                user_profile = get_user_profile_by_email(arg)
+                self.compute_activity(UserActivity.objects.filter(
+                        user_profile=user_profile))
+            except UserProfile.DoesNotExist:
+                try:
+                    # Report activity for a realm.
+                    realm = get_realm(arg)
+                    self.compute_activity(UserActivity.objects.filter(
+                            user_profile__realm=realm))
+                except Realm.DoesNotExist:
+                    print("Unknown user or domain %s" % (arg,))
+                    exit(1)

analytics/management/commands/populate_analytics_db.py

@@ -1,348 +0,0 @@
-import os
-from datetime import timedelta
-from typing import Any, Dict, List, Mapping, Type, Union
-
-from django.core.files.uploadedfile import UploadedFile
-from django.core.management.base import BaseCommand
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
-from analytics.lib.fixtures import generate_time_series_data
-from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    FillState,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-)
-from zerver.actions.create_realm import do_create_realm
-from zerver.actions.users import do_change_user_role
-from zerver.lib.create_user import create_user
-from zerver.lib.storage import static_path
-from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
-from zerver.lib.timestamp import floor_to_day
-from zerver.lib.upload import upload_message_attachment_from_request
-from zerver.models import (
-    Client,
-    Realm,
-    RealmAuditLog,
-    Recipient,
-    Stream,
-    Subscription,
-    UserGroup,
-    UserProfile,
-)
-
-
-class Command(BaseCommand):
-    help = """Populates analytics tables with randomly generated data."""
-
-    DAYS_OF_DATA = 100
-    random_seed = 26
-
-    def generate_fixture_data(
-        self,
-        stat: CountStat,
-        business_hours_base: float,
-        non_business_hours_base: float,
-        growth: float,
-        autocorrelation: float,
-        spikiness: float,
-        holiday_rate: float = 0,
-        partial_sum: bool = False,
-    ) -> List[int]:
-        self.random_seed += 1
-        return generate_time_series_data(
-            days=self.DAYS_OF_DATA,
-            business_hours_base=business_hours_base,
-            non_business_hours_base=non_business_hours_base,
-            growth=growth,
-            autocorrelation=autocorrelation,
-            spikiness=spikiness,
-            holiday_rate=holiday_rate,
-            frequency=stat.frequency,
-            partial_sum=partial_sum,
-            random_seed=self.random_seed,
-        )
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        # TODO: This should arguably only delete the objects
-        # associated with the "analytics" realm.
-        do_drop_all_analytics_tables()
-
-        # This also deletes any objects with this realm as a foreign key
-        Realm.objects.filter(string_id="analytics").delete()
-
-        # Because we just deleted a bunch of objects in the database
-        # directly (rather than deleting individual objects in Django,
-        # in which case our post_save hooks would have flushed the
-        # individual objects from memcached for us), we need to flush
-        # memcached in order to ensure deleted objects aren't still
-        # present in the memcached cache.
-        from zerver.apps import flush_cache
-
-        flush_cache(None)
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
-        realm = do_create_realm(
-            string_id="analytics", name="Analytics", date_created=installation_time
-        )
-
-        shylock = create_user(
-            "shylock@analytics.ds",
-            "Shylock",
-            realm,
-            full_name="Shylock",
-            role=UserProfile.ROLE_REALM_OWNER,
-            force_date_joined=installation_time,
-        )
-        do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None)
-
-        # Create guest user for set_guest_users_statistic.
-        create_user(
-            "bassanio@analytics.ds",
-            "Bassanio",
-            realm,
-            full_name="Bassanio",
-            role=UserProfile.ROLE_GUEST,
-            force_date_joined=installation_time,
-        )
-
-        administrators_user_group = UserGroup.objects.get(
-            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=realm, is_system_group=True
-        )
-        stream = Stream.objects.create(
-            name="all",
-            realm=realm,
-            date_created=installation_time,
-            can_remove_subscribers_group=administrators_user_group,
-        )
-        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
-        stream.recipient = recipient
-        stream.save(update_fields=["recipient"])
-
-        # Subscribe shylock to the stream to avoid invariant failures.
-        Subscription.objects.create(
-            recipient=recipient,
-            user_profile=shylock,
-            is_user_active=shylock.is_active,
-            color=STREAM_ASSIGNMENT_COLORS[0],
-        )
-        RealmAuditLog.objects.create(
-            realm=realm,
-            modified_user=shylock,
-            modified_stream=stream,
-            event_last_message_id=0,
-            event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
-            event_time=installation_time,
-        )
-
-        # Create an attachment in the database for set_storage_space_used_statistic.
-        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
-        file_info = os.stat(IMAGE_FILE_PATH)
-        file_size = file_info.st_size
-        with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock, file_size)
-
-        FixtureData = Mapping[Union[str, int, None], List[int]]
-
-        def insert_fixture_data(
-            stat: CountStat,
-            fixture_data: FixtureData,
-            table: Type[BaseCount],
-        ) -> None:
-            end_times = time_range(
-                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
-            )
-            if table == InstallationCount:
-                id_args: Dict[str, Any] = {}
-            if table == RealmCount:
-                id_args = {"realm": realm}
-            if table == UserCount:
-                id_args = {"realm": realm, "user": shylock}
-            if table == StreamCount:
-                id_args = {"stream": stream, "realm": realm}
-
-            for subgroup, values in fixture_data.items():
-                table.objects.bulk_create(
-                    table(
-                        property=stat.property,
-                        subgroup=subgroup,
-                        end_time=end_time,
-                        value=value,
-                        **id_args,
-                    )
-                    for end_time, value in zip(end_times, values)
-                    if value != 0
-                )
-
-        stat = COUNT_STATS["1day_actives::day"]
-        realm_data: FixtureData = {
-            None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data: FixtureData = {
-            None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["7day_actives::day"]
-        realm_data = {
-            None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["realm_active_humans::day"]
-        realm_data = {
-            None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        realm_data = {
-            "false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
-            "true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
-            "true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        user_data: FixtureData = {
-            "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_sent:message_type:day"]
-        user_data = {
-            "public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
-            "private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
-            "huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            "public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
-            "private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
-            "private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
-            "huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
-            "private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
-            "private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
-            "huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        website, created = Client.objects.get_or_create(name="website")
-        old_desktop, created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
-        android, created = Client.objects.get_or_create(name="ZulipAndroid")
-        iOS, created = Client.objects.get_or_create(name="ZulipiOS")
-        react_native, created = Client.objects.get_or_create(name="ZulipMobile")
-        API, created = Client.objects.get_or_create(name="API: Python")
-        zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
-        unused, created = Client.objects.get_or_create(name="unused")
-        long_webhook, created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
-
-        stat = COUNT_STATS["messages_sent:client:day"]
-        user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
-            android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
-        realm_data = {
-            "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data: Mapping[Union[int, str, None], List[int]] = {
-            "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
-        }
-        insert_fixture_data(stat, stream_data, StreamCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_read::hour"]
-        user_data = {
-            None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )

analytics/management/commands/realm_stats.py

@@ -0,0 +1,167 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+import datetime
+import pytz
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
+    Subscription, UserMessage, get_realm
+
+MOBILE_CLIENT_LIST = ["Android", "ios"]
+HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
+
+human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LIST)
+
+class Command(BaseCommand):
+    help = "Generate statistics on realm activity."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def active_users(self, realm):
+        # type: (Realm) -> List[UserProfile]
+        # Has been active (on the website, for now) in the last 7 days.
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
+        return [activity.user_profile for activity in \
+                    UserActivity.objects.filter(user_profile__realm=realm,
+                                                user_profile__is_active=True,
+                                                last_visit__gt=activity_cutoff,
+                                                query="/json/users/me/pointer",
+                                                client__name="website")]
+
+    def messages_sent_by(self, user, days_ago):
+        # type: (UserProfile, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
+
+    def total_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
+
+    def human_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
+
+    def api_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
+
+    def stream_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
+                                     recipient__type=Recipient.STREAM).count()
+
+    def private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
+            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
+
+    def group_private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
+            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
+
+    def report_percentage(self, numerator, denominator, text):
+        # type: (float, float, str) -> None
+        if not denominator:
+            fraction = 0.0
+        else:
+            fraction = numerator / float(denominator)
+        print("%.2f%% of" % (fraction * 100,), text)
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.domain)
+
+            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
+            active_users = self.active_users(realm)
+            num_active = len(active_users)
+
+            print("%d active users (%d total)" % (num_active, len(user_profiles)))
+            streams = Stream.objects.filter(realm=realm).extra(
+                tables=['zerver_subscription', 'zerver_recipient'],
+                where=['zerver_subscription.recipient_id = zerver_recipient.id',
+                       'zerver_recipient.type = 2',
+                       'zerver_recipient.type_id = zerver_stream.id',
+                       'zerver_subscription.active = true']).annotate(count=Count("name"))
+            print("%d streams" % (streams.count(),))
+
+            for days_ago in (1, 7, 30):
+                print("In last %d days, users sent:" % (days_ago,))
+                sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
+                for quantity in sorted(sender_quantities, reverse=True):
+                    print(quantity, end=' ')
+                print("")
+
+                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
+                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
+                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
+                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
+
+            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications == True])
+            self.report_percentage(num_notifications_enabled, num_active,
+                                   "active users have desktop notifications enabled")
+
+            num_enter_sends = len([x for x in active_users if x.enter_sends])
+            self.report_percentage(num_enter_sends, num_active,
+                                   "active users have enter-sends")
+
+            all_message_count = human_messages.filter(sender__realm=realm).count()
+            multi_paragraph_message_count = human_messages.filter(
+                sender__realm=realm, content__contains="\n\n").count()
+            self.report_percentage(multi_paragraph_message_count, all_message_count,
+                                   "all messages are multi-paragraph")
+
+            # Starred messages
+            starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
+                                                  flags=UserMessage.flags.starred).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users have starred %d messages" % (
+                len(starrers), sum([elt["count"] for elt in starrers])))
+
+            active_user_subs = Subscription.objects.filter(
+                user_profile__in=user_profiles, active=True)
+
+            # Streams not in home view
+            non_home_view = active_user_subs.filter(in_home_view=False).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users have %d streams not in home view" % (
+                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
+
+            # Code block markup
+            markup_messages = human_messages.filter(
+                sender__realm=realm, content__contains="~~~").values(
+                "sender").annotate(count=Count("sender"))
+            print("%d users have used code block markup on %s messages" % (
+                len(markup_messages), sum([elt["count"] for elt in markup_messages])))
+
+            # Notifications for stream messages
+            notifications = active_user_subs.filter(notifications=True).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users receive desktop notifications for %d streams" % (
+                len(notifications), sum([elt["count"] for elt in notifications])))
+
+            print("")

analytics/management/commands/stream_stats.py

@@ -0,0 +1,46 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
+
+class Command(BaseCommand):
+    help = "Generate statistics on the streams for a realm."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.domain)
+            print("------------")
+            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
+            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
+            invite_only_count = 0
+            for stream in streams:
+                if stream.invite_only:
+                    invite_only_count += 1
+                    continue
+                print("%25s" % (stream.name,), end=' ')
+                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
+                num_messages = len(Message.objects.filter(recipient=recipient))
+                print("%12d" % (num_messages,))
+            print("%d invite-only streams" % (invite_only_count,))
+            print("")

analytics/management/commands/update_analytics_counts.py

@@ -1,96 +0,0 @@
-import os
-import time
-from argparse import ArgumentParser
-from datetime import timezone
-from typing import Any, Dict
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-from django.utils.dateparse import parse_datetime
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
-from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.remote_server import send_analytics_to_remote_server
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import Realm
-
-
-class Command(BaseCommand):
-    help = """Fills Analytics tables.
-
-    Run as a cron job that runs every hour."""
-
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument(
-            "--time",
-            "-t",
-            help="Update stat tables from current state to "
-            "--time. Defaults to the current time.",
-            default=timezone_now().isoformat(),
-        )
-        parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
-        parser.add_argument(
-            "--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
-        )
-        parser.add_argument(
-            "--verbose", action="store_true", help="Print timing information to stdout."
-        )
-
-    def handle(self, *args: Any, **options: Any) -> None:
-        try:
-            os.mkdir(settings.ANALYTICS_LOCK_DIR)
-        except OSError:
-            print(
-                f"{WARNING}Analytics lock {settings.ANALYTICS_LOCK_DIR} is unavailable;"
-                f" exiting.{ENDC}"
-            )
-            return
-
-        try:
-            self.run_update_analytics_counts(options)
-        finally:
-            os.rmdir(settings.ANALYTICS_LOCK_DIR)
-
-    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
-        # installation_epoch relies on there being at least one realm; we
-        # shouldn't run the analytics code if that condition isn't satisfied
-        if not Realm.objects.exists():
-            logger.info("No realms, stopping update_analytics_counts")
-            return
-
-        fill_to_time = parse_datetime(options["time"])
-        assert fill_to_time is not None
-        if options["utc"]:
-            fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
-        if fill_to_time.tzinfo is None:
-            raise ValueError(
-                "--time must be time-zone-aware. Maybe you meant to use the --utc option?"
-            )
-
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
-
-        if options["stat"] is not None:
-            stats = [COUNT_STATS[options["stat"]]]
-        else:
-            stats = list(COUNT_STATS.values())
-
-        logger.info("Starting updating analytics counts through %s", fill_to_time)
-        if options["verbose"]:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options["verbose"]:
-                print(f"Updated {stat.property} in {time.time() - last:.3f}s")
-                last = time.time()
-
-        if options["verbose"]:
-            print(
-                f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
-            )
-        logger.info("Finished updating analytics counts through %s", fill_to_time)
-
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
-            send_analytics_to_remote_server()

analytics/management/commands/user_stats.py

@@ -0,0 +1,48 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from argparse import ArgumentParser
+import datetime
+import pytz
+from typing import Any
+
+from django.core.management.base import BaseCommand
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def messages_sent_by(self, user, week):
+        # type: (UserProfile, int) -> int
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
+        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(domain) for domain in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.domain)
+            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
+            print("%d users" % (len(user_profiles),))
+            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
+
+            for user_profile in user_profiles:
+                print("%35s" % (user_profile.email,), end=' ')
+                for week in range(10):
+                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
+                print("")

analytics/migrations/0001_initial.py

@@ -1,208 +0,0 @@
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("zerver", "0030_realm_org_type"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="Anomaly",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                ("info", models.CharField(max_length=1000)),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.CreateModel(
-            name="HuddleCount",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                (
-                    "huddle",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.CreateModel(
-            name="InstallationCount",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.CreateModel(
-            name="RealmCount",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.CreateModel(
-            name="StreamCount",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                (
-                    "stream",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.CreateModel(
-            name="UserCount",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
-            ],
-            bases=(models.Model,),
-        ),
-        migrations.AlterUniqueTogether(
-            name="usercount",
-            unique_together={("user", "property", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="streamcount",
-            unique_together={("stream", "property", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="realmcount",
-            unique_together={("realm", "property", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="installationcount",
-            unique_together={("property", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="huddlecount",
-            unique_together={("huddle", "property", "end_time", "interval")},
-        ),
-    ]

analytics/migrations/0002_remove_huddlecount.py

@@ -1,29 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="huddlecount",
-            unique_together=set(),
-        ),
-        migrations.RemoveField(
-            model_name="huddlecount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="huddlecount",
-            name="huddle",
-        ),
-        migrations.RemoveField(
-            model_name="huddlecount",
-            name="user",
-        ),
-        migrations.DeleteModel(
-            name="HuddleCount",
-        ),
-    ]

analytics/migrations/0003_fillstate.py

@@ -1,26 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0002_remove_huddlecount"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="FillState",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                ("property", models.CharField(unique=True, max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("state", models.PositiveSmallIntegerField()),
-                ("last_modified", models.DateTimeField(auto_now=True)),
-            ],
-            bases=(models.Model,),
-        ),
-    ]

analytics/migrations/0004_add_subgroup.py

@@ -1,30 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0003_fillstate"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="installationcount",
-            name="subgroup",
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name="realmcount",
-            name="subgroup",
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name="streamcount",
-            name="subgroup",
-            field=models.CharField(max_length=16, null=True),
-        ),
-        migrations.AddField(
-            model_name="usercount",
-            name="subgroup",
-            field=models.CharField(max_length=16, null=True),
-        ),
-    ]

analytics/migrations/0005_alter_field_size.py

@@ -1,50 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0004_add_subgroup"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="interval",
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name="installationcount",
-            name="property",
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="interval",
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="property",
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="interval",
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="property",
-            field=models.CharField(max_length=32),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="interval",
-            field=models.CharField(max_length=8),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="property",
-            field=models.CharField(max_length=32),
-        ),
-    ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -1,26 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0005_alter_field_size"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="installationcount",
-            unique_together={("property", "subgroup", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="realmcount",
-            unique_together={("realm", "property", "subgroup", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="streamcount",
-            unique_together={("stream", "property", "subgroup", "end_time", "interval")},
-        ),
-        migrations.AlterUniqueTogether(
-            name="usercount",
-            unique_together={("user", "property", "subgroup", "end_time", "interval")},
-        ),
-    ]

analytics/migrations/0007_remove_interval.py

@@ -1,43 +0,0 @@
-# Generated by Django 1.10.4 on 2017-01-16 20:50
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0006_add_subgroup_to_unique_constraints"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="installationcount",
-            unique_together={("property", "subgroup", "end_time")},
-        ),
-        migrations.RemoveField(
-            model_name="installationcount",
-            name="interval",
-        ),
-        migrations.AlterUniqueTogether(
-            name="realmcount",
-            unique_together={("realm", "property", "subgroup", "end_time")},
-        ),
-        migrations.RemoveField(
-            model_name="realmcount",
-            name="interval",
-        ),
-        migrations.AlterUniqueTogether(
-            name="streamcount",
-            unique_together={("stream", "property", "subgroup", "end_time")},
-        ),
-        migrations.RemoveField(
-            model_name="streamcount",
-            name="interval",
-        ),
-        migrations.AlterUniqueTogether(
-            name="usercount",
-            unique_together={("user", "property", "subgroup", "end_time")},
-        ),
-        migrations.RemoveField(
-            model_name="usercount",
-            name="interval",
-        ),
-    ]

analytics/migrations/0008_add_count_indexes.py

@@ -1,24 +0,0 @@
-# Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("zerver", "0050_userprofile_avatar_version"),
-        ("analytics", "0007_remove_interval"),
-    ]
-
-    operations = [
-        migrations.AlterIndexTogether(
-            name="realmcount",
-            index_together={("property", "end_time")},
-        ),
-        migrations.AlterIndexTogether(
-            name="streamcount",
-            index_together={("property", "realm", "end_time")},
-        ),
-        migrations.AlterIndexTogether(
-            name="usercount",
-            index_together={("property", "realm", "end_time")},
-        ),
-    ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,30 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def delete_messages_sent_to_stream_stat(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    property = "messages_sent_to_stream:is_bot"
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0008_add_count_indexes"),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,28 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def clear_message_sent_by_message_type_values(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    property = "messages_sent:message_type:day"
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,27 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0010_clear_messages_sent_values"),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]

analytics/migrations/0012_add_on_delete.py

@@ -1,41 +0,0 @@
-# Generated by Django 1.11.6 on 2018-01-29 08:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0011_clear_analytics_tables"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-    ]

analytics/migrations/0013_remove_anomaly.py

@@ -1,31 +0,0 @@
-# Generated by Django 1.11.18 on 2019-02-02 02:47
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0012_add_on_delete"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="installationcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="realmcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="streamcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="usercount",
-            name="anomaly",
-        ),
-        migrations.DeleteModel(
-            name="Anomaly",
-        ),
-    ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -1,16 +0,0 @@
-# Generated by Django 1.11.26 on 2020-01-27 04:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0013_remove_anomaly"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="fillstate",
-            name="last_modified",
-        ),
-    ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,64 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db.models import Count, Sum
-
-
-def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
-    """This is a preparatory migration for our Analytics tables.
-
-    The backstory is that Django's unique_together indexes do not properly
-    handle the subgroup=None corner case (allowing duplicate rows that have a
-    subgroup of None), which meant that in race conditions, rather than updating
-    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
-    create a duplicate row.
-
-    In the next migration, we'll add a proper constraint to fix this bug, but
-    we need to fix any existing problematic rows before we can add that constraint.
-
-    We fix this in an appropriate fashion for each type of CountStat object; mainly
-    this means deleting the extra rows, but for LoggingCountStat objects, we need to
-    additionally combine the sums.
-    """
-    count_tables = dict(
-        realm=apps.get_model("analytics", "RealmCount"),
-        user=apps.get_model("analytics", "UserCount"),
-        stream=apps.get_model("analytics", "StreamCount"),
-        installation=apps.get_model("analytics", "InstallationCount"),
-    )
-
-    for name, count_table in count_tables.items():
-        value = [name, "property", "end_time"]
-        if name == "installation":
-            value = ["property", "end_time"]
-        counts = (
-            count_table.objects.filter(subgroup=None)
-            .values(*value)
-            .annotate(Count("id"), Sum("value"))
-            .filter(id__count__gt=1)
-        )
-
-        for count in counts:
-            count.pop("id__count")
-            total_value = count.pop("value__sum")
-            duplicate_counts = list(count_table.objects.filter(**count))
-            first_count = duplicate_counts[0]
-            if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
-                # For LoggingCountStat objects, the right fix is to combine the totals;
-                # for other CountStat objects, we expect the duplicates to have the same value.
-                # And so all we need to do is delete them.
-                first_count.value = total_value
-                first_count.save()
-            to_cleanup = duplicate_counts[1:]
-            for duplicate_count in to_cleanup:
-                duplicate_count.delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0014_remove_fillstate_last_modified"),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_duplicate_counts, reverse_code=migrations.RunPython.noop),
-    ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -1,92 +0,0 @@
-# Generated by Django 2.2.10 on 2020-02-29 19:40
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0015_clear_duplicate_counts"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="installationcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="realmcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="streamcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="usercount",
-            unique_together=set(),
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/models.py

@@ -1,137 +0,0 @@
-import datetime
-
-from django.db import models
-from django.db.models import Q, UniqueConstraint
-
-from zerver.lib.timestamp import floor_to_day
-from zerver.models import Realm, Stream, UserProfile
-
-
-class FillState(models.Model):
-    property = models.CharField(max_length=40, unique=True)
-    end_time = models.DateTimeField()
-
-    # Valid states are {DONE, STARTED}
-    DONE = 1
-    STARTED = 2
-    state = models.PositiveSmallIntegerField()
-
-    def __str__(self) -> str:
-        return f"{self.property} {self.end_time} {self.state}"
-
-
-# The earliest/starting end_time in FillState
-# We assume there is at least one realm
-def installation_epoch() -> datetime.datetime:
-    earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
-        "date_created__min"
-    ]
-    return floor_to_day(earliest_realm_creation)
-
-
-class BaseCount(models.Model):
-    # Note: When inheriting from BaseCount, you may want to rearrange
-    # the order of the columns in the migration to make sure they
-    # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)
-    subgroup = models.CharField(max_length=16, null=True)
-    end_time = models.DateTimeField()
-    value = models.BigIntegerField()
-
-    class Meta:
-        abstract = True
-
-
-class InstallationCount(BaseCount):
-    class Meta:
-        # Handles invalid duplicate InstallationCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_installation_count",
-            ),
-            UniqueConstraint(
-                fields=["property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ]
-
-    def __str__(self) -> str:
-        return f"{self.property} {self.subgroup} {self.value}"
-
-
-class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
-
-    class Meta:
-        # Handles invalid duplicate RealmCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["realm", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_realm_count",
-            ),
-            UniqueConstraint(
-                fields=["realm", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ]
-        index_together = ["property", "end_time"]
-
-    def __str__(self) -> str:
-        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
-
-
-class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
-
-    class Meta:
-        # Handles invalid duplicate UserCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["user", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_user_count",
-            ),
-            UniqueConstraint(
-                fields=["user", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_user_count_null_subgroup",
-            ),
-        ]
-        # This index dramatically improves the performance of
-        # aggregating from users to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __str__(self) -> str:
-        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
-
-
-class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
-
-    class Meta:
-        # Handles invalid duplicate StreamCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["stream", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_stream_count",
-            ),
-            UniqueConstraint(
-                fields=["stream", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ]
-        # This index dramatically improves the performance of
-        # aggregating from streams to realms
-        index_together = ["property", "realm", "end_time"]
-
-    def __str__(self) -> str:
-        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"

analytics/tests/test_activity_views.py

@@ -1,48 +0,0 @@
-from unittest import mock
-
-from django.utils.timezone import now as timezone_now
-
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.models import Client, UserActivity, UserProfile, flush_per_request_caches
-
-
-class ActivityTest(ZulipTestCase):
-    @mock.patch("stripe.Customer.list", return_value=[])
-    def test_activity(self, unused_mock: mock.Mock) -> None:
-        self.login("hamlet")
-        client, _ = Client.objects.get_or_create(name="website")
-        query = "/json/messages/flags"
-        last_visit = timezone_now()
-        count = 150
-        for activity_user_profile in UserProfile.objects.all():
-            UserActivity.objects.get_or_create(
-                user_profile=activity_user_profile,
-                client=client,
-                query=query,
-                count=count,
-                last_visit=last_visit,
-            )
-
-        # Fails when not staff
-        result = self.client_get("/activity")
-        self.assertEqual(result.status_code, 302)
-
-        user_profile = self.example_user("hamlet")
-        user_profile.is_staff = True
-        user_profile.save(update_fields=["is_staff"])
-
-        flush_per_request_caches()
-        with self.assert_database_query_count(18):
-            result = self.client_get("/activity")
-            self.assertEqual(result.status_code, 200)
-
-        flush_per_request_caches()
-        with self.assert_database_query_count(8):
-            result = self.client_get("/realm_activity/zulip/")
-            self.assertEqual(result.status_code, 200)
-
-        iago = self.example_user("iago")
-        flush_per_request_caches()
-        with self.assert_database_query_count(5):
-            result = self.client_get(f"/user_activity/{iago.id}/")
-            self.assertEqual(result.status_code, 200)

analytics/tests/test_fixtures.py

@@ -1,40 +0,0 @@
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-from zerver.lib.test_classes import ZulipTestCase
-
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self) -> None:
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0
-        )
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1,
-            business_hours_base=2000,
-            non_business_hours_base=1500,
-            growth=2,
-            spikiness=0,
-            frequency=CountStat.HOUR,
-        )
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assert_length(data, 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500 * 2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1,
-            business_hours_base=2000,
-            non_business_hours_base=2000,
-            autocorrelation=1,
-            frequency=CountStat.HOUR,
-        )
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])

analytics/tests/test_stats_views.py

@@ -1,629 +0,0 @@
-from datetime import datetime, timedelta, timezone
-from typing import List, Optional
-
-from django.utils.timezone import now as timezone_now
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, UserCount
-from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client, get_realm
-
-
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
-        self.user = self.example_user("hamlet")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_guest_user_cant_access_stats(self) -> None:
-        self.user = self.example_user("polonius")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-        result = self.client_get("/json/analytics/chart_data")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-    def test_stats_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 404)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-
-class TestGetChartData(ZulipTestCase):
-    def setUp(self) -> None:
-        super().setUp()
-        self.realm = get_realm("zulip")
-        self.user = self.example_user("hamlet")
-        self.login_user(self.user)
-        self.end_times_hour = [
-            ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
-        ]
-        self.end_times_day = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
-        ]
-
-    def data(self, i: int) -> List[int]:
-        return [0, 0, i, 0]
-
-    def insert_data(
-        self, stat: CountStat, realm_subgroups: List[Optional[str]], user_subgroups: List[str]
-    ) -> None:
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create(
-            RealmCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
-        UserCount.objects.bulk_create(
-            UserCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=200 + i,
-                realm=self.realm,
-                user=self.user,
-            )
-            for i, subgroup in enumerate(user_subgroups)
-        )
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "_1day": self.data(100),
-                    "_15day": self.data(100),
-                    "all_time": self.data(100),
-                },
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_over_time(self) -> None:
-        stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        self.insert_data(stat, ["true", "false"], ["false"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "user": {"bot": self.data(0), "human": self.data(200)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_message_type(self) -> None:
-        stat = COUNT_STATS["messages_sent:message_type:day"]
-        self.insert_data(
-            stat, ["public_stream", "private_message"], ["public_stream", "private_stream"]
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "Public streams": self.data(100),
-                    "Private streams": self.data(0),
-                    "Direct messages": self.data(101),
-                    "Group direct messages": self.data(0),
-                },
-                "user": {
-                    "Public streams": self.data(200),
-                    "Private streams": self.data(201),
-                    "Direct messages": self.data(0),
-                    "Group direct messages": self.data(0),
-                },
-                "display_order": [
-                    "Direct messages",
-                    "Public streams",
-                    "Private streams",
-                    "Group direct messages",
-                ],
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_client(self) -> None:
-        stat = COUNT_STATS["messages_sent:client:day"]
-        client1 = Client.objects.create(name="client 1")
-        client2 = Client.objects.create(name="client 2")
-        client3 = Client.objects.create(name="client 3")
-        client4 = Client.objects.create(name="client 4")
-        self.insert_data(
-            stat,
-            [str(client4.id), str(client3.id), str(client2.id)],
-            [str(client3.id), str(client1.id)],
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "client 4": self.data(100),
-                    "client 3": self.data(101),
-                    "client 2": self.data(102),
-                },
-                "user": {"client 3": self.data(200), "client 1": self.data(201)},
-                "display_order": ["client 1", "client 2", "client 3", "client 4"],
-                "result": "success",
-            },
-        )
-
-    def test_messages_read_over_time(self) -> None:
-        stat = COUNT_STATS["messages_read::hour"]
-        self.insert_data(stat, [None], [])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_read_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"read": self.data(100)},
-                "user": {"read": self.data(0)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_include_empty_subgroups(self) -> None:
-        FillState.objects.create(
-            property="realm_active_humans::day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"_1day": [0], "_15day": [0], "all_time": [0]})
-        self.assertFalse("user" in data)
-
-        FillState.objects.create(
-            property="messages_sent:is_bot:hour",
-            end_time=self.end_times_hour[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"human": [0], "bot": [0]})
-        self.assertEqual(data["user"], {"human": [0], "bot": [0]})
-
-        FillState.objects.create(
-            property="messages_sent:message_type:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["everyone"],
-            {
-                "Public streams": [0],
-                "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-        self.assertEqual(
-            data["user"],
-            {
-                "Public streams": [0],
-                "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-
-        FillState.objects.create(
-            property="messages_sent:client:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {})
-        self.assertEqual(data["user"], {})
-
-    def test_start_and_end(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[1],
-                "end": end_time_timestamps[2],
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["end_times"], end_time_timestamps[1:3])
-        self.assertEqual(
-            data["everyone"], {"_1day": [0, 100], "_15day": [0, 100], "all_time": [0, 100]}
-        )
-
-        # start later then end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[2],
-                "end": end_time_timestamps[1],
-            },
-        )
-        self.assert_json_error_contains(result, "Start time is later than")
-
-    def test_min_length(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        # test min_length is too short to change anything
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 2}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["end_times"], [datetime_to_timestamp(dt) for dt in self.end_times_day]
-        )
-        self.assertEqual(
-            data["everyone"],
-            {"_1day": self.data(100), "_15day": self.data(100), "all_time": self.data(100)},
-        )
-        # test min_length larger than filled data
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 5}
-        )
-        data = self.assert_json_success(result)
-        end_times = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)
-        ]
-        self.assertEqual(data["end_times"], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(
-            data["everyone"],
-            {
-                "_1day": [0, *self.data(100)],
-                "_15day": [0, *self.data(100)],
-                "all_time": [0, *self.data(100)],
-            },
-        )
-
-    def test_non_existent_chart(self) -> None:
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "does_not_exist"})
-        self.assert_json_error_contains(result, "Unknown chart name")
-
-    def test_analytics_not_running(self) -> None:
-        realm = get_realm("zulip")
-
-        self.assertEqual(FillState.objects.count(), 0)
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(hours=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=5)
-        fill_state = FillState.objects.create(
-            property="messages_sent:is_bot:hour", end_time=end_time, state=FillState.DONE
-        )
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=2)
-        fill_state.end_time = end_time
-        fill_state.save(update_fields=["end_time"])
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/not_existing_realm",
-            {"chart_name": "number_of_humans"},
-        )
-        self.assert_json_error(result, "Invalid organization", 400)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    def test_sort_by_totals(self) -> None:
-        empty: List[int] = []
-        value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
-        self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
-
-    def test_sort_client_labels(self) -> None:
-        data = {
-            "everyone": {"a": [16], "c": [15], "b": [14], "e": [13], "d": [12], "h": [11]},
-            "user": {"a": [6], "b": [5], "d": [4], "e": [3], "f": [2], "g": [1]},
-        }
-        self.assertEqual(sort_client_labels(data), ["a", "b", "c", "d", "e", "f", "g", "h"])
-
-
-class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
-        HOUR = timedelta(hours=1)
-        DAY = timedelta(days=1)
-
-        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
-        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
-        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
-
-        # test start == end
-        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
-        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
-        # test start == end == boundary, and min_length == 0
-        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
-        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
-        # test start and end on different boundaries
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, None),
-            [floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, None),
-            [floor_day, floor_day + DAY],
-        )
-        # test min_length
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, 4),
-            [floor_hour - 2 * HOUR, floor_hour - HOUR, floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, 4),
-            [floor_day - 2 * DAY, floor_day - DAY, floor_day, floor_day + DAY],
-        )
-
-
-class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
-        a = {
-            "desktop app 1.0": [1, 2, 3],
-            "desktop app 2.0": [10, 12, 13],
-            "desktop app 3.0": [21, 22, 23],
-            "website": [1, 2, 3],
-            "ZulipiOS": [1, 2, 3],
-            "ZulipElectron": [2, 5, 7],
-            "ZulipMobile": [1, 5, 7],
-            "ZulipPython": [1, 2, 3],
-            "API: Python": [1, 2, 3],
-            "SomethingRandom": [4, 5, 6],
-            "ZulipGitHubWebhook": [7, 7, 9],
-            "ZulipAndroid": [64, 63, 65],
-            "ZulipTerminal": [9, 10, 11],
-        }
-        result = rewrite_client_arrays(a)
-        self.assertEqual(
-            result,
-            {
-                "Old desktop app": [32, 36, 39],
-                "Old iOS app": [1, 2, 3],
-                "Desktop app": [2, 5, 7],
-                "Mobile app": [1, 5, 7],
-                "Web app": [1, 2, 3],
-                "Python API": [2, 4, 6],
-                "SomethingRandom": [4, 5, 6],
-                "GitHub webhook": [7, 7, 9],
-                "Old Android app": [64, 63, 65],
-                "Terminal app": [9, 10, 11],
-            },
-        )

analytics/tests/test_support_views.py

@@ -1,741 +0,0 @@
-from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Optional
-from unittest import mock
-
-import orjson
-from django.utils.timezone import now as timezone_now
-
-from corporate.lib.stripe import add_months, update_sponsorship_status
-from corporate.models import Customer, CustomerPlan, LicenseLedger, get_customer_by_realm
-from zerver.actions.invites import do_create_multiuse_invite_link
-from zerver.actions.realm_settings import do_change_realm_org_type, do_send_realm_reactivation_email
-from zerver.actions.user_settings import do_change_user_setting
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm
-from zerver.models import (
-    MultiuseInvite,
-    PreregistrationUser,
-    Realm,
-    UserMessage,
-    UserProfile,
-    get_org_type_display_name,
-    get_realm,
-)
-
-if TYPE_CHECKING:
-    from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
-
-
-class TestSupportEndpoint(ZulipTestCase):
-    def test_search(self) -> None:
-        reset_email_visibility_to_everyone_in_zulip_realm()
-        lear_user = self.lear_user("king")
-        lear_user.is_staff = True
-        lear_user.save(update_fields=["is_staff"])
-        lear_realm = get_realm("lear")
-
-        def assert_user_details_in_html_response(
-            html_response: "TestHttpResponse", full_name: str, email: str, role: str
-        ) -> None:
-            self.assert_in_success_response(
-                [
-                    '<span class="label">user</span>\n',
-                    f"<h3>{full_name}</h3>",
-                    f"<b>Email</b>: {email}",
-                    "<b>Is active</b>: True<br />",
-                    f"<b>Role</b>: {role}<br />",
-                ],
-                html_response,
-            )
-
-        def create_invitation(
-            stream: str, invitee_email: str, realm: Optional[Realm] = None
-        ) -> None:
-            invite_expires_in_minutes = 10 * 24 * 60
-            self.client_post(
-                "/json/invites",
-                {
-                    "invitee_emails": [invitee_email],
-                    "stream_ids": orjson.dumps([self.get_stream_id(stream, realm)]).decode(),
-                    "invite_expires_in_minutes": invite_expires_in_minutes,
-                    "invite_as": PreregistrationUser.INVITE_AS["MEMBER"],
-                },
-                subdomain=realm.string_id if realm is not None else "zulip",
-            )
-
-        def check_hamlet_user_query_result(result: "TestHttpResponse") -> None:
-            assert_user_details_in_html_response(
-                result, "King Hamlet", self.example_email("hamlet"), "Member"
-            )
-            self.assert_in_success_response(
-                [
-                    f"<b>Admins</b>: {self.example_email('iago')}\n",
-                    f"<b>Owners</b>: {self.example_email('desdemona')}\n",
-                    'class="copy-button" data-copytext="{}">'.format(self.example_email("iago")),
-                    'class="copy-button" data-copytext="{}">'.format(
-                        self.example_email("desdemona")
-                    ),
-                ],
-                result,
-            )
-
-        def check_lear_user_query_result(result: "TestHttpResponse") -> None:
-            assert_user_details_in_html_response(
-                result, lear_user.full_name, lear_user.email, "Member"
-            )
-
-        def check_othello_user_query_result(result: "TestHttpResponse") -> None:
-            assert_user_details_in_html_response(
-                result, "Othello, the Moor of Venice", self.example_email("othello"), "Member"
-            )
-
-        def check_polonius_user_query_result(result: "TestHttpResponse") -> None:
-            assert_user_details_in_html_response(
-                result, "Polonius", self.example_email("polonius"), "Guest"
-            )
-
-        def check_zulip_realm_query_result(result: "TestHttpResponse") -> None:
-            zulip_realm = get_realm("zulip")
-            first_human_user = zulip_realm.get_first_human_user()
-            assert first_human_user is not None
-            self.assert_in_success_response(
-                [
-                    f"<b>First human user</b>: {first_human_user.delivery_email}\n",
-                    f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"',
-                    "Zulip Dev</h3>",
-                    '<option value="1" selected>Self-hosted</option>',
-                    '<option value="2" >Limited</option>',
-                    'input type="number" name="discount" value="None"',
-                    '<option value="active" selected>Active</option>',
-                    '<option value="deactivated" >Deactivated</option>',
-                    f'<option value="{zulip_realm.org_type}" selected>',
-                    'scrub-realm-button">',
-                    'data-string-id="zulip"',
-                ],
-                result,
-            )
-
-        def check_lear_realm_query_result(result: "TestHttpResponse") -> None:
-            self.assert_in_success_response(
-                [
-                    f'<input type="hidden" name="realm_id" value="{lear_realm.id}"',
-                    "Lear &amp; Co.</h3>",
-                    '<option value="1" selected>Self-hosted</option>',
-                    '<option value="2" >Limited</option>',
-                    'input type="number" name="discount" value="None"',
-                    '<option value="active" selected>Active</option>',
-                    '<option value="deactivated" >Deactivated</option>',
-                    'scrub-realm-button">',
-                    'data-string-id="lear"',
-                    "<b>Name</b>: Zulip Cloud Standard",
-                    "<b>Status</b>: Active",
-                    "<b>Billing schedule</b>: Annual",
-                    "<b>Licenses</b>: 2/10 (Manual)",
-                    "<b>Price per license</b>: $80.0",
-                    "<b>Next invoice date</b>: 02 January 2017",
-                    '<option value="send_invoice" selected>',
-                    '<option value="charge_automatically" >',
-                ],
-                result,
-            )
-
-        def check_preregistration_user_query_result(
-            result: "TestHttpResponse", email: str, invite: bool = False
-        ) -> None:
-            self.assert_in_success_response(
-                [
-                    '<span class="label">preregistration user</span>\n',
-                    f"<b>Email</b>: {email}",
-                ],
-                result,
-            )
-            if invite:
-                self.assert_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(
-                    [
-                        "<b>Expires in</b>: 1\xa0week, 3\xa0days",
-                        "<b>Status</b>: Link has not been used",
-                    ],
-                    result,
-                )
-                self.assert_in_success_response([], result)
-            else:
-                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
-                self.assert_in_success_response(
-                    [
-                        "<b>Expires in</b>: 1\xa0day",
-                        "<b>Status</b>: Link has not been used",
-                    ],
-                    result,
-                )
-
-        def check_realm_creation_query_result(result: "TestHttpResponse", email: str) -> None:
-            self.assert_in_success_response(
-                [
-                    '<span class="label">preregistration user</span>\n',
-                    '<span class="label">realm creation</span>\n',
-                    "<b>Link</b>: http://testserver/accounts/do_confirm/",
-                    "<b>Expires in</b>: 1\xa0day",
-                ],
-                result,
-            )
-
-        def check_multiuse_invite_link_query_result(result: "TestHttpResponse") -> None:
-            self.assert_in_success_response(
-                [
-                    '<span class="label">multiuse invite</span>\n',
-                    "<b>Link</b>: http://zulip.testserver/join/",
-                    "<b>Expires in</b>: 1\xa0week, 3\xa0days",
-                ],
-                result,
-            )
-
-        def check_realm_reactivation_link_query_result(result: "TestHttpResponse") -> None:
-            self.assert_in_success_response(
-                [
-                    '<span class="label">realm reactivation</span>\n',
-                    "<b>Link</b>: http://zulip.testserver/reactivate/",
-                    "<b>Expires in</b>: 1\xa0day",
-                ],
-                result,
-            )
-
-        def get_check_query_result(
-            query: str, count: int, subdomain: str = "zulip"
-        ) -> "TestHttpResponse":
-            result = self.client_get("/activity/support", {"q": query}, subdomain=subdomain)
-            self.assertEqual(result.content.decode().count("support-query-result"), count)
-            return result
-
-        self.login("cordelia")
-
-        result = self.client_get("/activity/support")
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        do_change_user_setting(
-            self.example_user("hamlet"),
-            "email_address_visibility",
-            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
-            acting_user=None,
-        )
-
-        customer = Customer.objects.create(realm=lear_realm, stripe_customer_id="cus_123")
-        now = datetime(2016, 1, 2, tzinfo=timezone.utc)
-        plan = CustomerPlan.objects.create(
-            customer=customer,
-            billing_cycle_anchor=now,
-            billing_schedule=CustomerPlan.ANNUAL,
-            tier=CustomerPlan.STANDARD,
-            price_per_license=8000,
-            next_invoice_date=add_months(now, 12),
-        )
-        LicenseLedger.objects.create(
-            licenses=10,
-            licenses_at_next_renewal=10,
-            event_time=timezone_now(),
-            is_renewal=True,
-            plan=plan,
-        )
-
-        result = self.client_get("/activity/support")
-        self.assert_in_success_response(
-            ['<input type="text" name="q" class="input-xxlarge search-query"'], result
-        )
-
-        result = get_check_query_result(self.example_email("hamlet"), 1)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        # Search should be case-insensitive:
-        assert self.example_email("hamlet") != self.example_email("hamlet").upper()
-        result = get_check_query_result(self.example_email("hamlet").upper(), 1)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        result = get_check_query_result(lear_user.email, 1)
-        check_lear_user_query_result(result)
-        check_lear_realm_query_result(result)
-
-        result = get_check_query_result(self.example_email("polonius"), 1)
-        check_polonius_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        result = get_check_query_result("lear", 1)
-        check_lear_realm_query_result(result)
-
-        result = get_check_query_result("http://lear.testserver", 1)
-        check_lear_realm_query_result(result)
-
-        with self.settings(REALM_HOSTS={"zulip": "localhost"}):
-            result = get_check_query_result("http://localhost", 1)
-            check_zulip_realm_query_result(result)
-
-        result = get_check_query_result("hamlet@zulip.com, lear", 2)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        result = get_check_query_result("King hamlet,lear", 2)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        result = get_check_query_result("Othello, the Moor of Venice", 1)
-        check_othello_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        result = get_check_query_result("lear, Hamlet <hamlet@zulip.com>", 2)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-        check_lear_realm_query_result(result)
-
-        with mock.patch(
-            "analytics.views.support.timezone_now",
-            return_value=timezone_now() - timedelta(minutes=50),
-        ):
-            self.client_post("/accounts/home/", {"email": self.nonreg_email("test")})
-            self.login("iago")
-            result = get_check_query_result(self.nonreg_email("test"), 1)
-            check_preregistration_user_query_result(result, self.nonreg_email("test"))
-            check_zulip_realm_query_result(result)
-
-            create_invitation("Denmark", self.nonreg_email("test1"))
-            result = get_check_query_result(self.nonreg_email("test1"), 1)
-            check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
-            check_zulip_realm_query_result(result)
-
-            email = self.nonreg_email("alice")
-            self.submit_realm_creation_form(
-                email, realm_subdomain="zuliptest", realm_name="Zulip test"
-            )
-            result = get_check_query_result(email, 1)
-            check_realm_creation_query_result(result, email)
-
-            invite_expires_in_minutes = 10 * 24 * 60
-            do_create_multiuse_invite_link(
-                self.example_user("hamlet"),
-                invited_as=1,
-                invite_expires_in_minutes=invite_expires_in_minutes,
-            )
-            result = get_check_query_result("zulip", 2)
-            check_multiuse_invite_link_query_result(result)
-            check_zulip_realm_query_result(result)
-            MultiuseInvite.objects.all().delete()
-
-            do_send_realm_reactivation_email(get_realm("zulip"), acting_user=None)
-            result = get_check_query_result("zulip", 2)
-            check_realm_reactivation_link_query_result(result)
-            check_zulip_realm_query_result(result)
-
-            lear_nonreg_email = "newguy@lear.org"
-            self.client_post("/accounts/home/", {"email": lear_nonreg_email}, subdomain="lear")
-            result = get_check_query_result(lear_nonreg_email, 1)
-            check_preregistration_user_query_result(result, lear_nonreg_email)
-            check_lear_realm_query_result(result)
-
-            self.login_user(lear_user)
-            create_invitation("general", "newguy2@lear.org", lear_realm)
-            result = get_check_query_result("newguy2@lear.org", 1, lear_realm.string_id)
-            check_preregistration_user_query_result(result, "newguy2@lear.org", invite=True)
-            check_lear_realm_query_result(result)
-
-    def test_get_org_type_display_name(self) -> None:
-        self.assertEqual(get_org_type_display_name(Realm.ORG_TYPES["business"]["id"]), "Business")
-        self.assertEqual(get_org_type_display_name(883), "")
-
-    def test_unspecified_org_type_correctly_displayed(self) -> None:
-        """
-        Unspecified org type is special in that it is marked to not be shown
-        on the registration page (because organitions are not meant to be able to choose it),
-        but should be correctly shown at the /support/ endpoint.
-        """
-        realm = get_realm("zulip")
-
-        do_change_realm_org_type(realm, 0, acting_user=None)
-        self.assertEqual(realm.org_type, 0)
-
-        self.login("iago")
-
-        result = self.client_get("/activity/support", {"q": "zulip"}, subdomain="zulip")
-        self.assert_in_success_response(
-            [
-                f'<input type="hidden" name="realm_id" value="{realm.id}"',
-                '<option value="0" selected>',
-            ],
-            result,
-        )
-
-    @mock.patch("analytics.views.support.update_billing_method_of_current_plan")
-    def test_change_billing_method(self, m: mock.Mock) -> None:
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{iago.realm_id}", "billing_method": "charge_automatically"},
-        )
-        m.assert_called_once_with(get_realm("zulip"), charge_automatically=True, acting_user=iago)
-        self.assert_in_success_response(
-            ["Billing method of zulip updated to charge automatically"], result
-        )
-
-        m.reset_mock()
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{iago.realm_id}", "billing_method": "send_invoice"}
-        )
-        m.assert_called_once_with(get_realm("zulip"), charge_automatically=False, acting_user=iago)
-        self.assert_in_success_response(
-            ["Billing method of zulip updated to pay by invoice"], result
-        )
-
-    def test_change_realm_plan_type(self) -> None:
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "2"}
-            )
-            m.assert_called_once_with(get_realm("zulip"), 2, acting_user=iago)
-            self.assert_in_success_response(
-                ["Plan type of zulip changed from self-hosted to limited"], result
-            )
-
-        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "10"}
-            )
-            m.assert_called_once_with(get_realm("zulip"), 10, acting_user=iago)
-            self.assert_in_success_response(
-                ["Plan type of zulip changed from self-hosted to plus"], result
-            )
-
-    def test_change_org_type(self) -> None:
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "org_type": "70"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        with mock.patch("analytics.views.support.do_change_realm_org_type") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{iago.realm_id}", "org_type": "70"}
-            )
-            m.assert_called_once_with(get_realm("zulip"), 70, acting_user=iago)
-            self.assert_in_success_response(
-                ["Org type of zulip changed from Business to Government"], result
-            )
-
-    def test_attach_discount(self) -> None:
-        cordelia = self.example_user("cordelia")
-        lear_realm = get_realm("lear")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.attach_discount_to_realm") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
-            )
-            m.assert_called_once_with(get_realm("lear"), 25, acting_user=iago)
-            self.assert_in_success_response(["Discount of lear changed to 25% from 0%"], result)
-
-    def test_change_sponsorship_status(self) -> None:
-        lear_realm = get_realm("lear")
-        self.assertIsNone(get_customer_by_realm(lear_realm))
-
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
-        )
-        self.assert_in_success_response(["lear marked as pending sponsorship."], result)
-        customer = get_customer_by_realm(lear_realm)
-        assert customer is not None
-        self.assertTrue(customer.sponsorship_pending)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "false"}
-        )
-        self.assert_in_success_response(["lear is no longer pending sponsorship."], result)
-        customer = get_customer_by_realm(lear_realm)
-        assert customer is not None
-        self.assertFalse(customer.sponsorship_pending)
-
-    def test_approve_sponsorship(self) -> None:
-        lear_realm = get_realm("lear")
-        update_sponsorship_status(lear_realm, True, acting_user=None)
-        king_user = self.lear_user("king")
-        king_user.role = UserProfile.ROLE_REALM_OWNER
-        king_user.save()
-
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
-        )
-        self.assert_in_success_response(["Sponsorship approved for lear"], result)
-        lear_realm.refresh_from_db()
-        self.assertEqual(lear_realm.plan_type, Realm.PLAN_TYPE_STANDARD_FREE)
-        customer = get_customer_by_realm(lear_realm)
-        assert customer is not None
-        self.assertFalse(customer.sponsorship_pending)
-        messages = UserMessage.objects.filter(user_profile=king_user)
-        self.assertIn(
-            "request for sponsored hosting has been approved", messages[0].message.content
-        )
-        self.assert_length(messages, 1)
-
-    def test_activate_or_deactivate_realm(self) -> None:
-        cordelia = self.example_user("cordelia")
-        lear_realm = get_realm("lear")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.do_deactivate_realm") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
-            )
-            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
-            self.assert_in_success_response(["lear deactivated"], result)
-
-        with mock.patch("analytics.views.support.do_send_realm_reactivation_email") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "active"}
-            )
-            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
-            self.assert_in_success_response(
-                ["Realm reactivation email sent to admins of lear"], result
-            )
-
-    def test_change_subdomain(self) -> None:
-        cordelia = self.example_user("cordelia")
-        lear_realm = get_realm("lear")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new_name"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-        self.login("iago")
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/activity/support?q=new-name")
-        realm_id = lear_realm.id
-        lear_realm = get_realm("new-name")
-        self.assertEqual(lear_realm.id, realm_id)
-        self.assertTrue(Realm.objects.filter(string_id="lear").exists())
-        self.assertTrue(Realm.objects.filter(string_id="lear")[0].deactivated)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
-        )
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "zulip"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
-        )
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "lear"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
-        )
-
-        # Test renaming to a "reserved" subdomain
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "your-org"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain reserved. Please choose a different one."], result
-        )
-
-    def test_downgrade_realm(self) -> None:
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        with mock.patch("analytics.views.support.downgrade_at_the_end_of_billing_cycle") as m:
-            result = self.client_post(
-                "/activity/support",
-                {
-                    "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "downgrade_at_billing_cycle_end",
-                },
-            )
-            m.assert_called_once_with(get_realm("zulip"))
-            self.assert_in_success_response(
-                ["zulip marked for downgrade at the end of billing cycle"], result
-            )
-
-        with mock.patch(
-            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
-        ) as m:
-            result = self.client_post(
-                "/activity/support",
-                {
-                    "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "downgrade_now_without_additional_licenses",
-                },
-            )
-            m.assert_called_once_with(get_realm("zulip"))
-            self.assert_in_success_response(
-                ["zulip downgraded without creating additional invoices"], result
-            )
-
-        with mock.patch(
-            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
-        ) as m1:
-            with mock.patch("analytics.views.support.void_all_open_invoices", return_value=1) as m2:
-                result = self.client_post(
-                    "/activity/support",
-                    {
-                        "realm_id": f"{iago.realm_id}",
-                        "modify_plan": "downgrade_now_void_open_invoices",
-                    },
-                )
-                m1.assert_called_once_with(get_realm("zulip"))
-                m2.assert_called_once_with(get_realm("zulip"))
-                self.assert_in_success_response(
-                    ["zulip downgraded and voided 1 open invoices"], result
-                )
-
-        with mock.patch("analytics.views.support.switch_realm_from_standard_to_plus_plan") as m:
-            result = self.client_post(
-                "/activity/support",
-                {
-                    "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "upgrade_to_plus",
-                },
-            )
-            m.assert_called_once_with(get_realm("zulip"))
-            self.assert_in_success_response(["zulip upgraded to Plus"], result)
-
-    def test_scrub_realm(self) -> None:
-        cordelia = self.example_user("cordelia")
-        lear_realm = get_realm("lear")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.do_scrub_realm") as m:
-            result = self.client_post(
-                "/activity/support", {"realm_id": f"{lear_realm.id}", "scrub_realm": "true"}
-            )
-            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
-            self.assert_in_success_response(["lear scrubbed"], result)
-
-        with mock.patch("analytics.views.support.do_scrub_realm") as m:
-            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
-            self.assert_json_error(result, "Invalid parameters")
-            m.assert_not_called()
-
-    def test_delete_user(self) -> None:
-        cordelia = self.example_user("cordelia")
-        hamlet = self.example_user("hamlet")
-        hamlet_email = hamlet.delivery_email
-        realm = get_realm("zulip")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.do_delete_user_preserving_messages") as m:
-            result = self.client_post(
-                "/activity/support",
-                {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id},
-            )
-            m.assert_called_once_with(hamlet)
-            self.assert_in_success_response([f"{hamlet_email} in zulip deleted"], result)

analytics/urls.py

@@ -1,69 +1,9 @@
-from typing import List, Union
+from django.conf.urls import patterns, url
 
-from django.conf.urls import include
-from django.urls import path
-from django.urls.resolvers import URLPattern, URLResolver
-
-from analytics.views.installation_activity import get_installation_activity
-from analytics.views.realm_activity import get_realm_activity
-from analytics.views.stats import (
-    get_chart_data,
-    get_chart_data_for_installation,
-    get_chart_data_for_realm,
-    get_chart_data_for_remote_installation,
-    get_chart_data_for_remote_realm,
-    stats,
-    stats_for_installation,
-    stats_for_realm,
-    stats_for_remote_installation,
-    stats_for_remote_realm,
-)
-from analytics.views.support import support
-from analytics.views.user_activity import get_user_activity
-from zerver.lib.rest import rest_path
-
-i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
-    # Server admin (user_profile.is_staff) visible stats pages
-    path("activity", get_installation_activity),
-    path("activity/support", support, name="support"),
-    path("realm_activity/<realm_str>/", get_realm_activity),
-    path("user_activity/<user_profile_id>/", get_user_activity),
-    path("stats/realm/<realm_str>/", stats_for_realm),
-    path("stats/installation", stats_for_installation),
-    path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
-    path(
-        "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
-    ),
-    # User-visible stats page
-    path("stats", stats, name="stats"),
+i18n_urlpatterns = [
+    url(r'^activity$', 'analytics.views.get_activity'),
+    url(r'^realm_activity/(?P<realm>[\S]+)/$', 'analytics.views.get_realm_activity'),
+    url(r'^user_activity/(?P<email>[\S]+)/$', 'analytics.views.get_user_activity'),
 ]
 
-# These endpoints are a part of the API (V1), which uses:
-# * REST verbs
-# * Basic auth (username:password is email:apiKey)
-# * Takes and returns json-formatted data
-#
-# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
-#
-# All of these paths are accessed by either a /json or /api prefix
-v1_api_and_json_patterns = [
-    # get data for the graphs at /stats
-    rest_path("analytics/chart_data", GET=get_chart_data),
-    rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
-    rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
-    rest_path(
-        "analytics/chart_data/remote/<int:remote_server_id>/installation",
-        GET=get_chart_data_for_remote_installation,
-    ),
-    rest_path(
-        "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
-        GET=get_chart_data_for_remote_realm,
-    ),
-]
-
-i18n_urlpatterns += [
-    path("api/v1/", include(v1_api_and_json_patterns)),
-    path("json/", include(v1_api_and_json_patterns)),
-]
-
-urlpatterns = i18n_urlpatterns
+urlpatterns = patterns('', *i18n_urlpatterns)

analytics/views.py

@@ -0,0 +1,925 @@
+from __future__ import absolute_import
+from __future__ import division
+from six import text_type
+from typing import Any, Dict, List, Tuple, Optional, Sequence, Callable, Union
+
+from django.db import connection
+from django.db.models.query import QuerySet
+from django.template import RequestContext, loader
+from django.core import urlresolvers
+from django.http import HttpResponseNotFound, HttpRequest, HttpResponse
+from jinja2 import Markup as mark_safe
+
+from zerver.decorator import has_request_variables, REQ, zulip_internal
+from zerver.models import get_realm, UserActivity, UserActivityInterval, Realm
+from zerver.lib.timestamp import timestamp_to_datetime
+
+from collections import defaultdict
+from datetime import datetime, timedelta
+import itertools
+import time
+import re
+import pytz
+from six.moves import filter
+from six.moves import map
+from six.moves import range
+from six.moves import zip
+eastern_tz = pytz.timezone('US/Eastern')
+
+from zproject.jinja2 import render_to_response
+
+def make_table(title, cols, rows, has_row_class=False):
+    # type: (str, List[str], List[Any], bool) -> str
+
+    if not has_row_class:
+        def fix_row(row):
+            # type: (Any) -> Dict[str, Any]
+            return dict(cells=row, row_class=None)
+        rows = list(map(fix_row, rows))
+
+    data = dict(title=title, cols=cols, rows=rows)
+
+    content = loader.render_to_string(
+        'analytics/ad_hoc_query.html',
+        dict(data=data)
+    )
+
+    return content
+
+def dictfetchall(cursor):
+    # type: (connection.cursor) -> List[Dict[str, Any]]
+    "Returns all rows from a cursor as a dict"
+    desc = cursor.description
+    return [
+        dict(list(zip([col[0] for col in desc], row)))
+        for row in cursor.fetchall()
+    ]
+
+
+def get_realm_day_counts():
+    # type: () -> Dict[str, Dict[str, str]]
+    query = '''
+        select
+            r.domain,
+            (now()::date - pub_date::date) age,
+            count(*) cnt
+        from zerver_message m
+        join zerver_userprofile up on up.id = m.sender_id
+        join zerver_realm r on r.id = up.realm_id
+        join zerver_client c on c.id = m.sending_client_id
+        where
+            (not up.is_bot)
+        and
+            pub_date > now()::date - interval '8 day'
+        and
+            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
+        group by
+            r.domain,
+            age
+        order by
+            r.domain,
+            age
+    '''
+    cursor = connection.cursor()
+    cursor.execute(query)
+    rows = dictfetchall(cursor)
+    cursor.close()
+
+    counts = defaultdict(dict) # type: Dict[str, Dict[int, int]]
+    for row in rows:
+        counts[row['domain']][row['age']] = row['cnt']
+
+
+    result = {}
+    for domain in counts:
+        raw_cnts = [counts[domain].get(age, 0) for age in range(8)]
+        min_cnt = min(raw_cnts)
+        max_cnt = max(raw_cnts)
+
+        def format_count(cnt):
+            # type: (int) -> str
+            if cnt == min_cnt:
+                good_bad = 'bad'
+            elif cnt == max_cnt:
+                good_bad = 'good'
+            else:
+                good_bad = 'neutral'
+
+            return '<td class="number %s">%s</td>' % (good_bad, cnt)
+
+        cnts = ''.join(map(format_count, raw_cnts))
+        result[domain] = dict(cnts=cnts)
+
+    return result
+
+def realm_summary_table(realm_minutes):
+    # type: (Dict[str, float]) -> str
+    query = '''
+        SELECT
+            realm.domain,
+            coalesce(user_counts.active_user_count, 0) active_user_count,
+            coalesce(at_risk_counts.at_risk_count, 0) at_risk_count,
+            (
+                SELECT
+                    count(*)
+                FROM zerver_userprofile up
+                WHERE up.realm_id = realm.id
+                AND is_active
+                AND not is_bot
+            ) user_profile_count,
+            (
+                SELECT
+                    count(*)
+                FROM zerver_userprofile up
+                WHERE up.realm_id = realm.id
+                AND is_active
+                AND is_bot
+            ) bot_count
+        FROM zerver_realm realm
+        LEFT OUTER JOIN
+            (
+                SELECT
+                    up.realm_id realm_id,
+                    count(distinct(ua.user_profile_id)) active_user_count
+                FROM zerver_useractivity ua
+                JOIN zerver_userprofile up
+                    ON up.id = ua.user_profile_id
+                WHERE
+                    query in (
+                        '/json/send_message',
+                        'send_message_backend',
+                        '/api/v1/send_message',
+                        '/json/update_pointer',
+                        '/json/users/me/pointer'
+                    )
+                AND
+                    last_visit > now() - interval '1 day'
+                AND
+                    not is_bot
+                GROUP BY realm_id
+            ) user_counts
+            ON user_counts.realm_id = realm.id
+        LEFT OUTER JOIN
+            (
+                SELECT
+                    realm_id,
+                    count(*) at_risk_count
+                FROM (
+                    SELECT
+                        realm.id as realm_id,
+                        up.email
+                    FROM zerver_useractivity ua
+                    JOIN zerver_userprofile up
+                        ON up.id = ua.user_profile_id
+                    JOIN zerver_realm realm
+                        ON realm.id = up.realm_id
+                    WHERE up.is_active
+                    AND (not up.is_bot)
+                    AND
+                        ua.query in (
+                            '/json/send_message',
+                            'send_message_backend',
+                            '/api/v1/send_message',
+                            '/json/update_pointer',
+                            '/json/users/me/pointer'
+                        )
+                    GROUP by realm.id, up.email
+                    HAVING max(last_visit) between
+                        now() - interval '7 day' and
+                        now() - interval '1 day'
+                ) as at_risk_users
+                GROUP BY realm_id
+            ) at_risk_counts
+            ON at_risk_counts.realm_id = realm.id
+        WHERE EXISTS (
+                SELECT *
+                FROM zerver_useractivity ua
+                JOIN zerver_userprofile up
+                    ON up.id = ua.user_profile_id
+                WHERE
+                    query in (
+                        '/json/send_message',
+                        '/api/v1/send_message',
+                        'send_message_backend',
+                        '/json/update_pointer',
+                        '/json/users/me/pointer'
+                    )
+                AND
+                    up.realm_id = realm.id
+                AND
+                    last_visit > now() - interval '2 week'
+        )
+        ORDER BY active_user_count DESC, domain ASC
+        '''
+
+    cursor = connection.cursor()
+    cursor.execute(query)
+    rows = dictfetchall(cursor)
+    cursor.close()
+
+    # get messages sent per day
+    counts = get_realm_day_counts()
+    for row in rows:
+        try:
+            row['history'] = counts[row['domain']]['cnts']
+        except:
+            row['history'] = ''
+
+    # augment data with realm_minutes
+    total_hours = 0.0
+    for row in rows:
+        domain = row['domain']
+        minutes = realm_minutes.get(domain, 0.0)
+        hours = minutes / 60.0
+        total_hours += hours
+        row['hours'] = str(int(hours))
+        try:
+            row['hours_per_user'] = '%.1f' % (hours / row['active_user_count'],)
+        except:
+            pass
+
+    # formatting
+    for row in rows:
+        row['domain'] = realm_activity_link(row['domain'])
+
+    # Count active sites
+    def meets_goal(row):
+        # type: (Dict[str, int]) -> bool
+        return row['active_user_count'] >= 5
+
+    num_active_sites = len(list(filter(meets_goal, rows)))
+
+    # create totals
+    total_active_user_count = 0
+    total_user_profile_count = 0
+    total_bot_count = 0
+    total_at_risk_count = 0
+    for row in rows:
+        total_active_user_count += int(row['active_user_count'])
+        total_user_profile_count += int(row['user_profile_count'])
+        total_bot_count += int(row['bot_count'])
+        total_at_risk_count += int(row['at_risk_count'])
+
+
+    rows.append(dict(
+        domain='Total',
+        active_user_count=total_active_user_count,
+        user_profile_count=total_user_profile_count,
+        bot_count=total_bot_count,
+        hours=int(total_hours),
+        at_risk_count=total_at_risk_count,
+    ))
+
+    content = loader.render_to_string(
+        'analytics/realm_summary_table.html',
+        dict(rows=rows, num_active_sites=num_active_sites)
+    )
+    return content
+
+
+def user_activity_intervals():
+    # type: () -> Tuple[mark_safe, Dict[str, float]]
+    day_end = timestamp_to_datetime(time.time())
+    day_start = day_end - timedelta(hours=24)
+
+    output = "Per-user online duration for the last 24 hours:\n"
+    total_duration = timedelta(0)
+
+    all_intervals = UserActivityInterval.objects.filter(
+        end__gte=day_start,
+        start__lte=day_end
+    ).select_related(
+        'user_profile',
+        'user_profile__realm'
+    ).only(
+        'start',
+        'end',
+        'user_profile__email',
+        'user_profile__realm__domain'
+    ).order_by(
+        'user_profile__realm__domain',
+        'user_profile__email'
+    )
+
+    by_domain = lambda row: row.user_profile.realm.domain
+    by_email = lambda row: row.user_profile.email
+
+    realm_minutes = {}
+
+    for domain, realm_intervals in itertools.groupby(all_intervals, by_domain):
+        realm_duration = timedelta(0)
+        output += '<hr>%s\n' % (domain,)
+        for email, intervals in itertools.groupby(realm_intervals, by_email):
+            duration = timedelta(0)
+            for interval in intervals:
+                start = max(day_start, interval.start)
+                end = min(day_end, interval.end)
+                duration += end - start
+
+            total_duration += duration
+            realm_duration += duration
+            output += "  %-*s%s\n" % (37, email, duration)
+
+        realm_minutes[domain] = realm_duration.total_seconds() / 60
+
+    output += "\nTotal Duration:                      %s\n" % (total_duration,)
+    output += "\nTotal Duration in minutes:           %s\n" % (total_duration.total_seconds() / 60.,)
+    output += "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
+    content = mark_safe('<pre>' + output + '</pre>')
+    return content, realm_minutes
+
+def sent_messages_report(realm):
+    # type: (str) -> str
+    title = 'Recently sent messages for ' + realm
+
+    cols = [
+        'Date',
+        'Humans',
+        'Bots'
+    ]
+
+    query = '''
+        select
+            series.day::date,
+            humans.cnt,
+            bots.cnt
+        from (
+            select generate_series(
+                (now()::date - interval '2 week'),
+                now()::date,
+                interval '1 day'
+            ) as day
+        ) as series
+        left join (
+            select
+                pub_date::date pub_date,
+                count(*) cnt
+            from zerver_message m
+            join zerver_userprofile up on up.id = m.sender_id
+            join zerver_realm r on r.id = up.realm_id
+            where
+                r.domain = %s
+            and
+                (not up.is_bot)
+            and
+                pub_date > now() - interval '2 week'
+            group by
+                pub_date::date
+            order by
+                pub_date::date
+        ) humans on
+            series.day = humans.pub_date
+        left join (
+            select
+                pub_date::date pub_date,
+                count(*) cnt
+            from zerver_message m
+            join zerver_userprofile up on up.id = m.sender_id
+            join zerver_realm r on r.id = up.realm_id
+            where
+                r.domain = %s
+            and
+                up.is_bot
+            and
+                pub_date > now() - interval '2 week'
+            group by
+                pub_date::date
+            order by
+                pub_date::date
+        ) bots on
+            series.day = bots.pub_date
+    '''
+    cursor = connection.cursor()
+    cursor.execute(query, [realm, realm])
+    rows = cursor.fetchall()
+    cursor.close()
+
+    return make_table(title, cols, rows)
+
+def ad_hoc_queries():
+    # type: () -> List[Dict[str, str]]
+    def get_page(query, cols, title):
+        # type: (str, List[str], str) -> Dict[str, str]
+        cursor = connection.cursor()
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        rows = list(map(list, rows))
+        cursor.close()
+
+        def fix_rows(i, fixup_func):
+            # type: (int, Union[Callable[[Realm], mark_safe], Callable[[datetime], str]]) -> None
+            for row in rows:
+                row[i] = fixup_func(row[i])
+
+        for i, col in enumerate(cols):
+            if col == 'Domain':
+                fix_rows(i, realm_activity_link)
+            elif col in ['Last time', 'Last visit']:
+                fix_rows(i, format_date_for_activity_reports)
+
+        content = make_table(title, cols, rows)
+
+        return dict(
+            content=content,
+            title=title
+        )
+
+    pages = []
+
+    ###
+
+    for mobile_type in ['Android', 'ZulipiOS']:
+        title = '%s usage' % (mobile_type,)
+
+        query = '''
+            select
+                realm.domain,
+                up.id user_id,
+                client.name,
+                sum(count) as hits,
+                max(last_visit) as last_time
+            from zerver_useractivity ua
+            join zerver_client client on client.id = ua.client_id
+            join zerver_userprofile up on up.id = ua.user_profile_id
+            join zerver_realm realm on realm.id = up.realm_id
+            where
+                client.name like '%s'
+            group by domain, up.id, client.name
+            having max(last_visit) > now() - interval '2 week'
+            order by domain, up.id, client.name
+        ''' % (mobile_type,)
+
+        cols = [
+            'Domain',
+            'User id',
+            'Name',
+            'Hits',
+            'Last time'
+        ]
+
+        pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = 'Desktop users'
+
+    query = '''
+        select
+            realm.domain,
+            client.name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            client.name like 'desktop%%'
+        group by domain, client.name
+        having max(last_visit) > now() - interval '2 week'
+        order by domain, client.name
+    '''
+
+    cols = [
+        'Domain',
+        'Client',
+        'Hits',
+        'Last time'
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = 'Integrations by domain'
+
+    query = '''
+        select
+            realm.domain,
+            case
+                when query like '%%external%%' then split_part(query, '/', 5)
+                else client.name
+            end client_name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            (query in ('send_message_backend', '/api/v1/send_message')
+            and client.name not in ('Android', 'ZulipiOS')
+            and client.name not like 'test: Zulip%%'
+            )
+        or
+            query like '%%external%%'
+        group by domain, client_name
+        having max(last_visit) > now() - interval '2 week'
+        order by domain, client_name
+    '''
+
+    cols = [
+        'Domain',
+        'Client',
+        'Hits',
+        'Last time'
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = 'Integrations by client'
+
+    query = '''
+        select
+            case
+                when query like '%%external%%' then split_part(query, '/', 5)
+                else client.name
+            end client_name,
+            realm.domain,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            (query in ('send_message_backend', '/api/v1/send_message')
+            and client.name not in ('Android', 'ZulipiOS')
+            and client.name not like 'test: Zulip%%'
+            )
+        or
+            query like '%%external%%'
+        group by client_name, domain
+        having max(last_visit) > now() - interval '2 week'
+        order by client_name, domain
+    '''
+
+    cols = [
+        'Client',
+        'Domain',
+        'Hits',
+        'Last time'
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    return pages
+
+@zulip_internal
+@has_request_variables
+def get_activity(request):
+    # type: (HttpRequest) -> HttpResponse
+    duration_content, realm_minutes = user_activity_intervals() # type: Tuple[mark_safe, Dict[str, float]]
+    counts_content = realm_summary_table(realm_minutes) # type: str
+    data = [
+        ('Counts', counts_content),
+        ('Durations', duration_content),
+    ]
+    for page in ad_hoc_queries():
+        data.append((page['title'], page['content']))
+
+    title = 'Activity'
+
+    return render_to_response(
+        'analytics/activity.html',
+        dict(data=data, title=title, is_home=True),
+        request=request
+    )
+
+def get_user_activity_records_for_realm(realm, is_bot):
+    # type: (str, bool) -> QuerySet
+    fields = [
+        'user_profile__full_name',
+        'user_profile__email',
+        'query',
+        'client__name',
+        'count',
+        'last_visit',
+    ]
+
+    records = UserActivity.objects.filter(
+            user_profile__realm__domain=realm,
+            user_profile__is_active=True,
+            user_profile__is_bot=is_bot
+    )
+    records = records.order_by("user_profile__email", "-last_visit")
+    records = records.select_related('user_profile', 'client').only(*fields)
+    return records
+
+def get_user_activity_records_for_email(email):
+    # type: (str) -> List[QuerySet]
+    fields = [
+        'user_profile__full_name',
+        'query',
+        'client__name',
+        'count',
+        'last_visit'
+    ]
+
+    records = UserActivity.objects.filter(
+            user_profile__email=email
+    )
+    records = records.order_by("-last_visit")
+    records = records.select_related('user_profile', 'client').only(*fields)
+    return records
+
+def raw_user_activity_table(records):
+    # type: (List[QuerySet]) -> str
+    cols = [
+        'query',
+        'client',
+        'count',
+        'last_visit'
+    ]
+
+    def row(record):
+        # type: (QuerySet) -> List[Any]
+        return [
+                record.query,
+                record.client.name,
+                record.count,
+                format_date_for_activity_reports(record.last_visit)
+        ]
+
+    rows = list(map(row, records))
+    title = 'Raw Data'
+    return make_table(title, cols, rows)
+
+def get_user_activity_summary(records):
+    # type: (List[QuerySet]) -> Dict[str, Dict[str, Any]]
+    #: `Any` used above should be `Union(int, datetime)`.
+    #: However current version of `Union` does not work inside other function.
+    #: We could use something like:
+    # `Union[Dict[str, Dict[str, int]], Dict[str, Dict[str, datetime]]]`
+    #: but that would require this long `Union` to carry on throughout inner functions.
+    summary = {} # type: Dict[str, Dict[str, Any]]
+    def update(action, record):
+        # type: (str, QuerySet) -> None
+        if action not in summary:
+            summary[action] = dict(
+                    count=record.count,
+                    last_visit=record.last_visit
+            )
+        else:
+            summary[action]['count'] += record.count
+            summary[action]['last_visit'] = max(
+                    summary[action]['last_visit'],
+                    record.last_visit
+            )
+
+    if records:
+        summary['name'] = records[0].user_profile.full_name
+
+    for record in records:
+        client = record.client.name
+        query = record.query
+
+        update('use', record)
+
+        if client == 'API':
+            m = re.match('/api/.*/external/(.*)', query)
+            if m:
+                client = m.group(1)
+                update(client, record)
+
+        if client.startswith('desktop'):
+            update('desktop', record)
+        if client == 'website':
+            update('website', record)
+        if ('send_message' in query) or re.search('/api/.*/external/.*', query):
+            update('send', record)
+        if query in ['/json/update_pointer', '/json/users/me/pointer', '/api/v1/update_pointer']:
+            update('pointer', record)
+        update(client, record)
+
+
+    return summary
+
+def format_date_for_activity_reports(date):
+    # type: (Optional[datetime]) -> str
+    if date:
+        return date.astimezone(eastern_tz).strftime('%Y-%m-%d %H:%M')
+    else:
+        return ''
+
+def user_activity_link(email):
+    # type: (str) -> mark_safe
+    url_name = 'analytics.views.get_user_activity'
+    url = urlresolvers.reverse(url_name, kwargs=dict(email=email))
+    email_link = '<a href="%s">%s</a>' % (url, email)
+    return mark_safe(email_link)
+
+def realm_activity_link(realm):
+    # type: (str) -> mark_safe
+    url_name = 'analytics.views.get_realm_activity'
+    url = urlresolvers.reverse(url_name, kwargs=dict(realm=realm))
+    realm_link = '<a href="%s">%s</a>' % (url, realm)
+    return mark_safe(realm_link)
+
+def realm_client_table(user_summaries):
+    # type: (Dict[str, Dict[str, Dict[str, Any]]]) -> str
+    exclude_keys = [
+            'internal',
+            'name',
+            'use',
+            'send',
+            'pointer',
+            'website',
+            'desktop',
+    ]
+
+    rows = []
+    for email, user_summary in user_summaries.items():
+        email_link = user_activity_link(email)
+        name = user_summary['name']
+        for k, v in user_summary.items():
+            if k in exclude_keys:
+                continue
+            client = k
+            count = v['count']
+            last_visit = v['last_visit']
+            row = [
+                    format_date_for_activity_reports(last_visit),
+                    client,
+                    name,
+                    email_link,
+                    count,
+            ]
+            rows.append(row)
+
+    rows = sorted(rows, key=lambda r: r[0], reverse=True)
+
+    cols = [
+            'Last visit',
+            'Client',
+            'Name',
+            'Email',
+            'Count',
+    ]
+
+    title = 'Clients'
+
+    return make_table(title, cols, rows)
+
+def user_activity_summary_table(user_summary):
+    # type: (Dict[str, Dict[str, Any]]) -> str
+    rows = []
+    for k, v in user_summary.items():
+        if k == 'name':
+            continue
+        client = k
+        count = v['count']
+        last_visit = v['last_visit']
+        row = [
+                format_date_for_activity_reports(last_visit),
+                client,
+                count,
+        ]
+        rows.append(row)
+
+    rows = sorted(rows, key=lambda r: r[0], reverse=True)
+
+    cols = [
+            'last_visit',
+            'client',
+            'count',
+    ]
+
+    title = 'User Activity'
+    return make_table(title, cols, rows)
+
+def realm_user_summary_table(all_records, admin_emails):
+    # type: (List[QuerySet], Set[text_type]) -> Tuple[Dict[str, Dict[str, Any]], str]
+    user_records = {}
+
+    def by_email(record):
+        # type: (QuerySet) -> str
+        return record.user_profile.email
+
+    for email, records in itertools.groupby(all_records, by_email):
+        user_records[email] = get_user_activity_summary(list(records))
+
+    def get_last_visit(user_summary, k):
+        # type: (Dict[str, Dict[str, datetime]], str) -> Optional[datetime]
+        if k in user_summary:
+            return user_summary[k]['last_visit']
+        else:
+            return None
+
+    def get_count(user_summary, k):
+        # type: (Dict[str, Dict[str, str]], str) -> str
+        if k in user_summary:
+            return user_summary[k]['count']
+        else:
+            return ''
+
+    def is_recent(val):
+        # type: (Optional[datetime]) -> bool
+        age = datetime.now(val.tzinfo) - val # type: ignore # datetie.now tzinfo bug.
+        return age.total_seconds() < 5 * 60
+
+    rows = []
+    for email, user_summary in user_records.items():
+        email_link = user_activity_link(email)
+        sent_count = get_count(user_summary, 'send')
+        cells = [user_summary['name'], email_link, sent_count]
+        row_class = ''
+        for field in ['use', 'send', 'pointer', 'desktop', 'ZulipiOS', 'Android']:
+            visit = get_last_visit(user_summary, field)
+            if field == 'use':
+                if visit and is_recent(visit):
+                    row_class += ' recently_active'
+                if email in admin_emails:
+                    row_class += ' admin'
+            val = format_date_for_activity_reports(visit)
+            cells.append(val)
+        row = dict(cells=cells, row_class=row_class)
+        rows.append(row)
+
+    def by_used_time(row):
+        # type: (Dict[str, Sequence[str]]) -> str
+        return row['cells'][3]
+
+    rows = sorted(rows, key=by_used_time, reverse=True)
+
+    cols = [
+            'Name',
+            'Email',
+            'Total sent',
+            'Heard from',
+            'Message sent',
+            'Pointer motion',
+            'Desktop',
+            'ZulipiOS',
+            'Android'
+    ]
+
+    title = 'Summary'
+
+    content = make_table(title, cols, rows, has_row_class=True)
+    return user_records, content
+
+@zulip_internal
+def get_realm_activity(request, realm):
+    # type: (HttpRequest, str) -> HttpResponse
+    data = [] # type: List[Tuple[str, str]]
+    all_user_records = {} # type: Dict[str, Any]
+
+    try:
+        admins = get_realm(realm).get_admin_users()
+    except Realm.DoesNotExist:
+        return HttpResponseNotFound("Realm %s does not exist" % (realm,))
+
+    admin_emails = {admin.email for admin in admins}
+
+    for is_bot, page_title in [(False,  'Humans'), (True, 'Bots')]:
+        all_records = list(get_user_activity_records_for_realm(realm, is_bot))
+
+        user_records, content = realm_user_summary_table(all_records, admin_emails)
+        all_user_records.update(user_records)
+
+        data += [(page_title, content)]
+
+    page_title = 'Clients'
+    content = realm_client_table(all_user_records)
+    data += [(page_title, content)]
+
+
+    page_title = 'History'
+    content = sent_messages_report(realm)
+    data += [(page_title, content)]
+
+    fix_name = lambda realm: realm.replace('.', '_')
+
+    realm_link = 'https://stats1.zulip.net:444/render/?from=-7days'
+    realm_link += '&target=stats.gauges.staging.users.active.%s.0_16hr' % (fix_name(realm),)
+
+    title = realm
+    return render_to_response(
+        'analytics/activity.html',
+        dict(data=data, realm_link=realm_link, title=title),
+        request=request
+    )
+
+@zulip_internal
+def get_user_activity(request, email):
+    # type: (HttpRequest, str) -> HttpResponse
+    records = get_user_activity_records_for_email(email)
+
+    data = [] # type: List[Tuple[str, str]]
+    user_summary = get_user_activity_summary(records)
+    content = user_activity_summary_table(user_summary)
+
+    data += [('Summary', content)]
+
+    content = raw_user_activity_table(records)
+    data += [('Info', content)]
+
+    title = email
+    return render_to_response(
+        'analytics/activity.html',
+        dict(data=data, title=title),
+        request=request
+    )

analytics/views/activity_common.py

@@ -1,154 +0,0 @@
-import re
-import sys
-from datetime import datetime
-from typing import Any, Collection, Dict, List, Optional, Sequence
-from urllib.parse import urlencode
-
-from django.conf import settings
-from django.db.backends.utils import CursorWrapper
-from django.template import loader
-from django.urls import reverse
-from markupsafe import Markup
-
-from zerver.lib.url_encoding import append_url_query_string
-from zerver.models import UserActivity, get_realm
-
-if sys.version_info < (3, 9):  # nocoverage
-    from backports import zoneinfo
-else:  # nocoverage
-    import zoneinfo
-
-eastern_tz = zoneinfo.ZoneInfo("America/New_York")
-
-
-if settings.BILLING_ENABLED:
-    pass
-
-
-def make_table(
-    title: str, cols: Sequence[str], rows: Sequence[Any], has_row_class: bool = False
-) -> str:
-    if not has_row_class:
-
-        def fix_row(row: Any) -> Dict[str, Any]:
-            return dict(cells=row, row_class=None)
-
-        rows = list(map(fix_row, rows))
-
-    data = dict(title=title, cols=cols, rows=rows)
-
-    content = loader.render_to_string(
-        "analytics/ad_hoc_query.html",
-        dict(data=data),
-    )
-
-    return content
-
-
-def dictfetchall(cursor: CursorWrapper) -> List[Dict[str, Any]]:
-    """Returns all rows from a cursor as a dict"""
-    desc = cursor.description
-    return [dict(zip((col[0] for col in desc), row)) for row in cursor.fetchall()]
-
-
-def format_date_for_activity_reports(date: Optional[datetime]) -> str:
-    if date:
-        return date.astimezone(eastern_tz).strftime("%Y-%m-%d %H:%M")
-    else:
-        return ""
-
-
-def user_activity_link(email: str, user_profile_id: int) -> Markup:
-    from analytics.views.user_activity import get_user_activity
-
-    url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
-    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
-
-
-def realm_activity_link(realm_str: str) -> Markup:
-    from analytics.views.realm_activity import get_realm_activity
-
-    url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
-
-
-def realm_stats_link(realm_str: str) -> Markup:
-    from analytics.views.stats import stats_for_realm
-
-    url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
-
-
-def realm_support_link(realm_str: str) -> Markup:
-    support_url = reverse("support")
-    query = urlencode({"q": realm_str})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
-
-
-def realm_url_link(realm_str: str) -> Markup:
-    url = get_realm(realm_str).uri
-    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
-
-
-def remote_installation_stats_link(server_id: int, hostname: str) -> Markup:
-    from analytics.views.stats import stats_for_remote_installation
-
-    url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i>{hostname}</a>').format(
-        url=url, hostname=hostname
-    )
-
-
-def get_user_activity_summary(records: Collection[UserActivity]) -> Dict[str, Any]:
-    #: The type annotation used above is clearly overly permissive.
-    #: We should perhaps use TypedDict to clearly lay out the schema
-    #: for the user activity summary.
-    summary: Dict[str, Any] = {}
-
-    def update(action: str, record: UserActivity) -> None:
-        if action not in summary:
-            summary[action] = dict(
-                count=record.count,
-                last_visit=record.last_visit,
-            )
-        else:
-            summary[action]["count"] += record.count
-            summary[action]["last_visit"] = max(
-                summary[action]["last_visit"],
-                record.last_visit,
-            )
-
-    if records:
-        first_record = next(iter(records))
-        summary["name"] = first_record.user_profile.full_name
-        summary["user_profile_id"] = first_record.user_profile.id
-
-    for record in records:
-        client = record.client.name
-        query = str(record.query)
-
-        update("use", record)
-
-        if client == "API":
-            m = re.match("/api/.*/external/(.*)", query)
-            if m:
-                client = m.group(1)
-                update(client, record)
-
-        if client.startswith("desktop"):
-            update("desktop", record)
-        if client == "website":
-            update("website", record)
-        if ("send_message" in query) or re.search("/api/.*/external/.*", query):
-            update("send", record)
-        if query in [
-            "/json/update_pointer",
-            "/json/users/me/pointer",
-            "/api/v1/update_pointer",
-            "update_pointer_backend",
-        ]:
-            update("pointer", record)
-        update(client, record)
-
-    return summary

analytics/views/installation_activity.py

@@ -1,620 +0,0 @@
-import itertools
-import time
-from collections import defaultdict
-from contextlib import suppress
-from datetime import datetime, timedelta
-from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
-
-from django.conf import settings
-from django.db import connection
-from django.http import HttpRequest, HttpResponse
-from django.shortcuts import render
-from django.template import loader
-from django.utils.timezone import now as timezone_now
-from markupsafe import Markup
-from psycopg2.sql import SQL, Composable, Literal
-
-from analytics.lib.counts import COUNT_STATS
-from analytics.views.activity_common import (
-    dictfetchall,
-    format_date_for_activity_reports,
-    make_table,
-    realm_activity_link,
-    realm_stats_link,
-    realm_support_link,
-    realm_url_link,
-    remote_installation_stats_link,
-)
-from analytics.views.support import get_plan_name
-from zerver.decorator import require_server_admin
-from zerver.lib.request import has_request_variables
-from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Realm, UserActivityInterval, get_org_type_display_name
-
-if settings.BILLING_ENABLED:
-    from corporate.lib.stripe import (
-        estimate_annual_recurring_revenue_by_realm,
-        get_realms_to_default_discount_dict,
-    )
-
-
-def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
-    query = SQL(
-        """
-        select
-            r.string_id,
-            (now()::date - date_sent::date) age,
-            count(*) cnt
-        from zerver_message m
-        join zerver_userprofile up on up.id = m.sender_id
-        join zerver_realm r on r.id = up.realm_id
-        join zerver_client c on c.id = m.sending_client_id
-        where
-            (not up.is_bot)
-        and
-            date_sent > now()::date - interval '8 day'
-        and
-            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
-        group by
-            r.string_id,
-            age
-        order by
-            r.string_id,
-            age
-    """
-    )
-    cursor = connection.cursor()
-    cursor.execute(query)
-    rows = dictfetchall(cursor)
-    cursor.close()
-
-    counts: Dict[str, Dict[int, int]] = defaultdict(dict)
-    for row in rows:
-        counts[row["string_id"]][row["age"]] = row["cnt"]
-
-    result = {}
-    for string_id in counts:
-        raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
-        min_cnt = min(raw_cnts[1:])
-        max_cnt = max(raw_cnts[1:])
-
-        def format_count(cnt: int, style: Optional[str] = None) -> Markup:
-            if style is not None:
-                good_bad = style
-            elif cnt == min_cnt:
-                good_bad = "bad"
-            elif cnt == max_cnt:
-                good_bad = "good"
-            else:
-                good_bad = "neutral"
-
-            return Markup('<td class="number {good_bad}">{cnt}</td>').format(
-                good_bad=good_bad, cnt=cnt
-            )
-
-        cnts = format_count(raw_cnts[0], "neutral") + Markup().join(map(format_count, raw_cnts[1:]))
-        result[string_id] = dict(cnts=cnts)
-
-    return result
-
-
-def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
-    now = timezone_now()
-
-    query = SQL(
-        """
-        SELECT
-            realm.string_id,
-            realm.date_created,
-            realm.plan_type,
-            realm.org_type,
-            coalesce(wau_table.value, 0) wau_count,
-            coalesce(dau_table.value, 0) dau_count,
-            coalesce(user_count_table.value, 0) user_profile_count,
-            coalesce(bot_count_table.value, 0) bot_count
-        FROM
-            zerver_realm as realm
-            LEFT OUTER JOIN (
-                SELECT
-                    value _14day_active_humans,
-                    realm_id
-                from
-                    analytics_realmcount
-                WHERE
-                    property = 'realm_active_humans::day'
-                    AND end_time = %(realm_active_humans_end_time)s
-            ) as _14day_active_humans_table ON realm.id = _14day_active_humans_table.realm_id
-            LEFT OUTER JOIN (
-                SELECT
-                    value,
-                    realm_id
-                from
-                    analytics_realmcount
-                WHERE
-                    property = '7day_actives::day'
-                    AND end_time = %(seven_day_actives_end_time)s
-            ) as wau_table ON realm.id = wau_table.realm_id
-            LEFT OUTER JOIN (
-                SELECT
-                    value,
-                    realm_id
-                from
-                    analytics_realmcount
-                WHERE
-                    property = '1day_actives::day'
-                    AND end_time = %(one_day_actives_end_time)s
-            ) as dau_table ON realm.id = dau_table.realm_id
-            LEFT OUTER JOIN (
-                SELECT
-                    value,
-                    realm_id
-                from
-                    analytics_realmcount
-                WHERE
-                    property = 'active_users_audit:is_bot:day'
-                    AND subgroup = 'false'
-                    AND end_time = %(active_users_audit_end_time)s
-            ) as user_count_table ON realm.id = user_count_table.realm_id
-            LEFT OUTER JOIN (
-                SELECT
-                    value,
-                    realm_id
-                from
-                    analytics_realmcount
-                WHERE
-                    property = 'active_users_audit:is_bot:day'
-                    AND subgroup = 'true'
-                    AND end_time = %(active_users_audit_end_time)s
-            ) as bot_count_table ON realm.id = bot_count_table.realm_id
-        WHERE
-            _14day_active_humans IS NOT NULL
-            or realm.plan_type = 3
-        ORDER BY
-            dau_count DESC,
-            string_id ASC
-    """
-    )
-
-    cursor = connection.cursor()
-    cursor.execute(
-        query,
-        {
-            "realm_active_humans_end_time": COUNT_STATS[
-                "realm_active_humans::day"
-            ].last_successful_fill(),
-            "seven_day_actives_end_time": COUNT_STATS["7day_actives::day"].last_successful_fill(),
-            "one_day_actives_end_time": COUNT_STATS["1day_actives::day"].last_successful_fill(),
-            "active_users_audit_end_time": COUNT_STATS[
-                "active_users_audit:is_bot:day"
-            ].last_successful_fill(),
-        },
-    )
-    rows = dictfetchall(cursor)
-    cursor.close()
-
-    for row in rows:
-        row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
-        row["age_days"] = int((now - row["date_created"]).total_seconds() / 86400)
-        row["is_new"] = row["age_days"] < 12 * 7
-
-    # get messages sent per day
-    counts = get_realm_day_counts()
-    for row in rows:
-        try:
-            row["history"] = counts[row["string_id"]]["cnts"]
-        except Exception:
-            row["history"] = ""
-
-    # estimate annual subscription revenue
-    total_arr = 0
-    if settings.BILLING_ENABLED:
-        estimated_arrs = estimate_annual_recurring_revenue_by_realm()
-        realms_to_default_discount = get_realms_to_default_discount_dict()
-
-        for row in rows:
-            row["plan_type_string"] = get_plan_name(row["plan_type"])
-
-            string_id = row["string_id"]
-
-            if string_id in estimated_arrs:
-                row["arr"] = estimated_arrs[string_id]
-
-            if row["plan_type"] in [Realm.PLAN_TYPE_STANDARD, Realm.PLAN_TYPE_PLUS]:
-                row["effective_rate"] = 100 - int(realms_to_default_discount.get(string_id, 0))
-            elif row["plan_type"] == Realm.PLAN_TYPE_STANDARD_FREE:
-                row["effective_rate"] = 0
-            elif (
-                row["plan_type"] == Realm.PLAN_TYPE_LIMITED
-                and string_id in realms_to_default_discount
-            ):
-                row["effective_rate"] = 100 - int(realms_to_default_discount[string_id])
-            else:
-                row["effective_rate"] = ""
-
-        total_arr += sum(estimated_arrs.values())
-
-    for row in rows:
-        row["org_type_string"] = get_org_type_display_name(row["org_type"])
-
-    # augment data with realm_minutes
-    total_hours = 0.0
-    for row in rows:
-        string_id = row["string_id"]
-        minutes = realm_minutes.get(string_id, 0.0)
-        hours = minutes / 60.0
-        total_hours += hours
-        row["hours"] = str(int(hours))
-        with suppress(Exception):
-            row["hours_per_user"] = "{:.1f}".format(hours / row["dau_count"])
-
-    # formatting
-    for row in rows:
-        row["realm_url"] = realm_url_link(row["string_id"])
-        row["stats_link"] = realm_stats_link(row["string_id"])
-        row["support_link"] = realm_support_link(row["string_id"])
-        row["string_id"] = realm_activity_link(row["string_id"])
-
-    # Count active sites
-    def meets_goal(row: Dict[str, int]) -> bool:
-        return row["dau_count"] >= 5
-
-    num_active_sites = len(list(filter(meets_goal, rows)))
-
-    # create totals
-    total_dau_count = 0
-    total_user_profile_count = 0
-    total_bot_count = 0
-    total_wau_count = 0
-    for row in rows:
-        total_dau_count += int(row["dau_count"])
-        total_user_profile_count += int(row["user_profile_count"])
-        total_bot_count += int(row["bot_count"])
-        total_wau_count += int(row["wau_count"])
-
-    total_row = dict(
-        string_id="Total",
-        plan_type_string="",
-        org_type_string="",
-        effective_rate="",
-        arr=total_arr,
-        realm_url="",
-        stats_link="",
-        support_link="",
-        date_created_day="",
-        dau_count=total_dau_count,
-        user_profile_count=total_user_profile_count,
-        bot_count=total_bot_count,
-        hours=int(total_hours),
-        wau_count=total_wau_count,
-    )
-
-    rows.insert(0, total_row)
-
-    content = loader.render_to_string(
-        "analytics/realm_summary_table.html",
-        dict(
-            rows=rows,
-            num_active_sites=num_active_sites,
-            utctime=now.strftime("%Y-%m-%d %H:%M %Z"),
-            billing_enabled=settings.BILLING_ENABLED,
-        ),
-    )
-    return content
-
-
-def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
-    day_end = timestamp_to_datetime(time.time())
-    day_start = day_end - timedelta(hours=24)
-
-    output = Markup()
-    output += "Per-user online duration for the last 24 hours:\n"
-    total_duration = timedelta(0)
-
-    all_intervals = (
-        UserActivityInterval.objects.filter(
-            end__gte=day_start,
-            start__lte=day_end,
-        )
-        .select_related(
-            "user_profile",
-            "user_profile__realm",
-        )
-        .only(
-            "start",
-            "end",
-            "user_profile__delivery_email",
-            "user_profile__realm__string_id",
-        )
-        .order_by(
-            "user_profile__realm__string_id",
-            "user_profile__delivery_email",
-        )
-    )
-
-    by_string_id = lambda row: row.user_profile.realm.string_id
-    by_email = lambda row: row.user_profile.delivery_email
-
-    realm_minutes = {}
-
-    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
-        realm_duration = timedelta(0)
-        output += Markup("<hr>") + f"{string_id}\n"
-        for email, intervals in itertools.groupby(realm_intervals, by_email):
-            duration = timedelta(0)
-            for interval in intervals:
-                start = max(day_start, interval.start)
-                end = min(day_end, interval.end)
-                duration += end - start
-
-            total_duration += duration
-            realm_duration += duration
-            output += f"  {email:<37}{duration}\n"
-
-        realm_minutes[string_id] = realm_duration.total_seconds() / 60
-
-    output += f"\nTotal duration:                      {total_duration}\n"
-    output += f"\nTotal duration in minutes:           {total_duration.total_seconds() / 60.}\n"
-    output += f"Total duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}"
-    content = Markup("<pre>{}</pre>").format(output)
-    return content, realm_minutes
-
-
-def ad_hoc_queries() -> List[Dict[str, str]]:
-    def get_page(
-        query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
-    ) -> Dict[str, str]:
-        cursor = connection.cursor()
-        cursor.execute(query)
-        rows = cursor.fetchall()
-        rows = list(map(list, rows))
-        cursor.close()
-
-        def fix_rows(
-            i: int, fixup_func: Union[Callable[[str], Markup], Callable[[datetime], str]]
-        ) -> None:
-            for row in rows:
-                row[i] = fixup_func(row[i])
-
-        total_row = []
-        for i, col in enumerate(cols):
-            if col == "Realm":
-                fix_rows(i, realm_activity_link)
-            elif col in ["Last time", "Last visit"]:
-                fix_rows(i, format_date_for_activity_reports)
-            elif col == "Hostname":
-                for row in rows:
-                    row[i] = remote_installation_stats_link(row[0], row[i])
-            if len(totals_columns) > 0:
-                if i == 0:
-                    total_row.append("Total")
-                elif i in totals_columns:
-                    total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
-                else:
-                    total_row.append("")
-        if len(totals_columns) > 0:
-            rows.insert(0, total_row)
-
-        content = make_table(title, cols, rows)
-
-        return dict(
-            content=content,
-            title=title,
-        )
-
-    pages = []
-
-    ###
-
-    for mobile_type in ["Android", "ZulipiOS"]:
-        title = f"{mobile_type} usage"
-
-        query: Composable = SQL(
-            """
-            select
-                realm.string_id,
-                up.id user_id,
-                client.name,
-                sum(count) as hits,
-                max(last_visit) as last_time
-            from zerver_useractivity ua
-            join zerver_client client on client.id = ua.client_id
-            join zerver_userprofile up on up.id = ua.user_profile_id
-            join zerver_realm realm on realm.id = up.realm_id
-            where
-                client.name like {mobile_type}
-            group by string_id, up.id, client.name
-            having max(last_visit) > now() - interval '2 week'
-            order by string_id, up.id, client.name
-        """
-        ).format(
-            mobile_type=Literal(mobile_type),
-        )
-
-        cols = [
-            "Realm",
-            "User id",
-            "Name",
-            "Hits",
-            "Last time",
-        ]
-
-        pages.append(get_page(query, cols, title))
-
-    ###
-
-    title = "Desktop users"
-
-    query = SQL(
-        """
-        select
-            realm.string_id,
-            client.name,
-            sum(count) as hits,
-            max(last_visit) as last_time
-        from zerver_useractivity ua
-        join zerver_client client on client.id = ua.client_id
-        join zerver_userprofile up on up.id = ua.user_profile_id
-        join zerver_realm realm on realm.id = up.realm_id
-        where
-            client.name like 'desktop%%'
-        group by string_id, client.name
-        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client.name
-    """
-    )
-
-    cols = [
-        "Realm",
-        "Client",
-        "Hits",
-        "Last time",
-    ]
-
-    pages.append(get_page(query, cols, title))
-
-    ###
-
-    title = "Integrations by realm"
-
-    query = SQL(
-        """
-        select
-            realm.string_id,
-            case
-                when query like '%%external%%' then split_part(query, '/', 5)
-                else client.name
-            end client_name,
-            sum(count) as hits,
-            max(last_visit) as last_time
-        from zerver_useractivity ua
-        join zerver_client client on client.id = ua.client_id
-        join zerver_userprofile up on up.id = ua.user_profile_id
-        join zerver_realm realm on realm.id = up.realm_id
-        where
-            (query in ('send_message_backend', '/api/v1/send_message')
-            and client.name not in ('Android', 'ZulipiOS')
-            and client.name not like 'test: Zulip%%'
-            )
-        or
-            query like '%%external%%'
-        group by string_id, client_name
-        having max(last_visit) > now() - interval '2 week'
-        order by string_id, client_name
-    """
-    )
-
-    cols = [
-        "Realm",
-        "Client",
-        "Hits",
-        "Last time",
-    ]
-
-    pages.append(get_page(query, cols, title))
-
-    ###
-
-    title = "Integrations by client"
-
-    query = SQL(
-        """
-        select
-            case
-                when query like '%%external%%' then split_part(query, '/', 5)
-                else client.name
-            end client_name,
-            realm.string_id,
-            sum(count) as hits,
-            max(last_visit) as last_time
-        from zerver_useractivity ua
-        join zerver_client client on client.id = ua.client_id
-        join zerver_userprofile up on up.id = ua.user_profile_id
-        join zerver_realm realm on realm.id = up.realm_id
-        where
-            (query in ('send_message_backend', '/api/v1/send_message')
-            and client.name not in ('Android', 'ZulipiOS')
-            and client.name not like 'test: Zulip%%'
-            )
-        or
-            query like '%%external%%'
-        group by client_name, string_id
-        having max(last_visit) > now() - interval '2 week'
-        order by client_name, string_id
-    """
-    )
-
-    cols = [
-        "Client",
-        "Realm",
-        "Hits",
-        "Last time",
-    ]
-
-    pages.append(get_page(query, cols, title))
-
-    title = "Remote Zulip servers"
-
-    query = SQL(
-        """
-        with icount as (
-            select
-                server_id,
-                max(value) as max_value,
-                max(end_time) as max_end_time
-            from zilencer_remoteinstallationcount
-            where
-                property='active_users:is_bot:day'
-                and subgroup='false'
-            group by server_id
-            ),
-        remote_push_devices as (
-            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
-            group by server_id
-        )
-        select
-            rserver.id,
-            rserver.hostname,
-            rserver.contact_email,
-            max_value,
-            push_user_count,
-            max_end_time
-        from zilencer_remotezulipserver rserver
-        left join icount on icount.server_id = rserver.id
-        left join remote_push_devices on remote_push_devices.server_id = rserver.id
-        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
-    """
-    )
-
-    cols = [
-        "ID",
-        "Hostname",
-        "Contact email",
-        "Analytics users",
-        "Mobile users",
-        "Last update time",
-    ]
-
-    pages.append(get_page(query, cols, title, totals_columns=[3, 4]))
-
-    return pages
-
-
-@require_server_admin
-@has_request_variables
-def get_installation_activity(request: HttpRequest) -> HttpResponse:
-    duration_content, realm_minutes = user_activity_intervals()
-    counts_content: str = realm_summary_table(realm_minutes)
-    data = [
-        ("Counts", counts_content),
-        ("Durations", duration_content),
-    ]
-    for page in ad_hoc_queries():
-        data.append((page["title"], page["content"]))
-
-    title = "Activity"
-
-    return render(
-        request,
-        "analytics/activity.html",
-        context=dict(data=data, title=title, is_home=True),
-    )

analytics/views/realm_activity.py

@@ -1,262 +0,0 @@
-import itertools
-from datetime import datetime
-from typing import Any, Dict, List, Optional, Set, Tuple
-
-from django.db import connection
-from django.db.models import QuerySet
-from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
-from django.shortcuts import render
-from django.utils.timezone import now as timezone_now
-from psycopg2.sql import SQL
-
-from analytics.views.activity_common import (
-    format_date_for_activity_reports,
-    get_user_activity_summary,
-    make_table,
-    realm_stats_link,
-    user_activity_link,
-)
-from zerver.decorator import require_server_admin
-from zerver.models import Realm, UserActivity
-
-
-def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet[UserActivity]:
-    fields = [
-        "user_profile__full_name",
-        "user_profile__delivery_email",
-        "query",
-        "client__name",
-        "count",
-        "last_visit",
-    ]
-
-    records = UserActivity.objects.filter(
-        user_profile__realm__string_id=realm,
-        user_profile__is_active=True,
-        user_profile__is_bot=is_bot,
-    )
-    records = records.order_by("user_profile__delivery_email", "-last_visit")
-    records = records.select_related("user_profile", "client").only(*fields)
-    return records
-
-
-def realm_user_summary_table(
-    all_records: QuerySet[UserActivity], admin_emails: Set[str]
-) -> Tuple[Dict[str, Any], str]:
-    user_records = {}
-
-    def by_email(record: UserActivity) -> str:
-        return record.user_profile.delivery_email
-
-    for email, records in itertools.groupby(all_records, by_email):
-        user_records[email] = get_user_activity_summary(list(records))
-
-    def get_last_visit(user_summary: Dict[str, Dict[str, datetime]], k: str) -> Optional[datetime]:
-        if k in user_summary:
-            return user_summary[k]["last_visit"]
-        else:
-            return None
-
-    def get_count(user_summary: Dict[str, Dict[str, str]], k: str) -> str:
-        if k in user_summary:
-            return user_summary[k]["count"]
-        else:
-            return ""
-
-    def is_recent(val: datetime) -> bool:
-        age = timezone_now() - val
-        return age.total_seconds() < 5 * 60
-
-    rows = []
-    for email, user_summary in user_records.items():
-        email_link = user_activity_link(email, user_summary["user_profile_id"])
-        sent_count = get_count(user_summary, "send")
-        cells = [user_summary["name"], email_link, sent_count]
-        row_class = ""
-        for field in ["use", "send", "pointer", "desktop", "ZulipiOS", "Android"]:
-            visit = get_last_visit(user_summary, field)
-            if field == "use":
-                if visit and is_recent(visit):
-                    row_class += " recently_active"
-                if email in admin_emails:
-                    row_class += " admin"
-            val = format_date_for_activity_reports(visit)
-            cells.append(val)
-        row = dict(cells=cells, row_class=row_class)
-        rows.append(row)
-
-    def by_used_time(row: Dict[str, Any]) -> str:
-        return row["cells"][3]
-
-    rows = sorted(rows, key=by_used_time, reverse=True)
-
-    cols = [
-        "Name",
-        "Email",
-        "Total sent",
-        "Heard from",
-        "Message sent",
-        "Pointer motion",
-        "Desktop",
-        "ZulipiOS",
-        "Android",
-    ]
-
-    title = "Summary"
-
-    content = make_table(title, cols, rows, has_row_class=True)
-    return user_records, content
-
-
-def realm_client_table(user_summaries: Dict[str, Dict[str, Any]]) -> str:
-    exclude_keys = [
-        "internal",
-        "name",
-        "user_profile_id",
-        "use",
-        "send",
-        "pointer",
-        "website",
-        "desktop",
-    ]
-
-    rows = []
-    for email, user_summary in user_summaries.items():
-        email_link = user_activity_link(email, user_summary["user_profile_id"])
-        name = user_summary["name"]
-        for k, v in user_summary.items():
-            if k in exclude_keys:
-                continue
-            client = k
-            count = v["count"]
-            last_visit = v["last_visit"]
-            row = [
-                format_date_for_activity_reports(last_visit),
-                client,
-                name,
-                email_link,
-                count,
-            ]
-            rows.append(row)
-
-    rows = sorted(rows, key=lambda r: r[0], reverse=True)
-
-    cols = [
-        "Last visit",
-        "Client",
-        "Name",
-        "Email",
-        "Count",
-    ]
-
-    title = "Clients"
-
-    return make_table(title, cols, rows)
-
-
-def sent_messages_report(realm: str) -> str:
-    title = "Recently sent messages for " + realm
-
-    cols = [
-        "Date",
-        "Humans",
-        "Bots",
-    ]
-
-    query = SQL(
-        """
-        select
-            series.day::date,
-            humans.cnt,
-            bots.cnt
-        from (
-            select generate_series(
-                (now()::date - interval '2 week'),
-                now()::date,
-                interval '1 day'
-            ) as day
-        ) as series
-        left join (
-            select
-                date_sent::date date_sent,
-                count(*) cnt
-            from zerver_message m
-            join zerver_userprofile up on up.id = m.sender_id
-            join zerver_realm r on r.id = up.realm_id
-            where
-                r.string_id = %s
-            and
-                (not up.is_bot)
-            and
-                date_sent > now() - interval '2 week'
-            group by
-                date_sent::date
-            order by
-                date_sent::date
-        ) humans on
-            series.day = humans.date_sent
-        left join (
-            select
-                date_sent::date date_sent,
-                count(*) cnt
-            from zerver_message m
-            join zerver_userprofile up on up.id = m.sender_id
-            join zerver_realm r on r.id = up.realm_id
-            where
-                r.string_id = %s
-            and
-                up.is_bot
-            and
-                date_sent > now() - interval '2 week'
-            group by
-                date_sent::date
-            order by
-                date_sent::date
-        ) bots on
-            series.day = bots.date_sent
-    """
-    )
-    cursor = connection.cursor()
-    cursor.execute(query, [realm, realm])
-    rows = cursor.fetchall()
-    cursor.close()
-
-    return make_table(title, cols, rows)
-
-
-@require_server_admin
-def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
-    data: List[Tuple[str, str]] = []
-    all_user_records: Dict[str, Any] = {}
-
-    try:
-        admins = Realm.objects.get(string_id=realm_str).get_human_admin_users()
-    except Realm.DoesNotExist:
-        return HttpResponseNotFound()
-
-    admin_emails = {admin.delivery_email for admin in admins}
-
-    for is_bot, page_title in [(False, "Humans"), (True, "Bots")]:
-        all_records = get_user_activity_records_for_realm(realm_str, is_bot)
-
-        user_records, content = realm_user_summary_table(all_records, admin_emails)
-        all_user_records.update(user_records)
-
-        data += [(page_title, content)]
-
-    page_title = "Clients"
-    content = realm_client_table(all_user_records)
-    data += [(page_title, content)]
-
-    page_title = "History"
-    content = sent_messages_report(realm_str)
-    data += [(page_title, content)]
-
-    title = realm_str
-    realm_stats = realm_stats_link(realm_str)
-
-    return render(
-        request,
-        "analytics/activity.html",
-        context=dict(data=data, realm_stats_link=realm_stats, title=title),
-    )

analytics/views/stats.py

@@ -1,542 +0,0 @@
-import logging
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
-
-from django.conf import settings
-from django.db.models import QuerySet
-from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
-from django.shortcuts import render
-from django.utils import translation
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-    installation_epoch,
-)
-from zerver.decorator import (
-    require_non_guest_user,
-    require_server_admin,
-    require_server_admin_api,
-    to_utc_datetime,
-    zulip_login_required,
-)
-from zerver.lib.exceptions import JsonableError
-from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
-from zerver.lib.request import REQ, has_request_variables
-from zerver.lib.response import json_success
-from zerver.lib.timestamp import convert_to_UTC
-from zerver.lib.validator import to_non_negative_int
-from zerver.models import Client, Realm, UserProfile, get_realm
-
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
-
-MAX_TIME_FOR_FULL_ANALYTICS_GENERATION = timedelta(days=1, minutes=30)
-
-
-def is_analytics_ready(realm: Realm) -> bool:
-    return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
-
-
-def render_stats(
-    request: HttpRequest,
-    data_url_suffix: str,
-    realm: Optional[Realm],
-    *,
-    title: Optional[str] = None,
-    for_installation: bool = False,
-    remote: bool = False,
-    analytics_ready: bool = True,
-) -> HttpResponse:
-    assert request.user.is_authenticated
-
-    if realm is not None:
-        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
-        guest_users = UserProfile.objects.filter(
-            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
-        ).count()
-        space_used = realm.currently_used_upload_space_bytes()
-        if title:
-            pass
-        else:
-            title = realm.name or realm.string_id
-    else:
-        assert title
-        guest_users = None
-        space_used = None
-
-    page_params = dict(
-        data_url_suffix=data_url_suffix,
-        for_installation=for_installation,
-        remote=remote,
-        upload_space_used=space_used,
-        guest_users=guest_users,
-    )
-
-    request_language = get_and_set_request_language(
-        request,
-        request.user.default_language,
-        translation.get_language_from_path(request.path_info),
-    )
-
-    page_params["translation_data"] = get_language_translation_data(request_language)
-
-    return render(
-        request,
-        "analytics/stats.html",
-        context=dict(
-            target_name=title,
-            page_params=page_params,
-            analytics_ready=analytics_ready,
-        ),
-    )
-
-
-@zulip_login_required
-def stats(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    realm = request.user.realm
-    if request.user.is_guest:
-        # TODO: Make @zulip_login_required pass the UserProfile so we
-        # can use @require_member_or_admin
-        raise JsonableError(_("Not allowed for guest users"))
-    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
-
-
-@require_server_admin
-@has_request_variables
-def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        return HttpResponseNotFound()
-
-    return render_stats(
-        request,
-        f"/realm/{realm_str}",
-        realm,
-        analytics_ready=is_analytics_ready(realm),
-    )
-
-
-@require_server_admin
-@has_request_variables
-def stats_for_remote_realm(
-    request: HttpRequest, remote_server_id: int, remote_realm_id: int
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/realm/{remote_realm_id}",
-        None,
-        title=f"Realm {remote_realm_id} on server {server.hostname}",
-    )
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_realm(
-    request: HttpRequest, /, user_profile: UserProfile, realm_str: str, **kwargs: Any
-) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        raise JsonableError(_("Invalid organization"))
-
-    return get_chart_data(request, user_profile, realm=realm, **kwargs)
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_remote_realm(
-    request: HttpRequest,
-    /,
-    user_profile: UserProfile,
-    remote_server_id: int,
-    remote_realm_id: int,
-    **kwargs: Any,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return get_chart_data(
-        request,
-        user_profile,
-        server=server,
-        remote=True,
-        remote_realm_id=int(remote_realm_id),
-        **kwargs,
-    )
-
-
-@require_server_admin
-def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation", for_installation=True)
-
-
-@require_server_admin
-def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/installation",
-        None,
-        title=f"remote installation {server.hostname}",
-        for_installation=True,
-        remote=True,
-    )
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_installation(
-    request: HttpRequest, /, user_profile: UserProfile, chart_name: str = REQ(), **kwargs: Any
-) -> HttpResponse:
-    return get_chart_data(request, user_profile, for_installation=True, **kwargs)
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_remote_installation(
-    request: HttpRequest,
-    /,
-    user_profile: UserProfile,
-    remote_server_id: int,
-    chart_name: str = REQ(),
-    **kwargs: Any,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return get_chart_data(
-        request,
-        user_profile,
-        for_installation=True,
-        remote=True,
-        server=server,
-        **kwargs,
-    )
-
-
-@require_non_guest_user
-@has_request_variables
-def get_chart_data(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    chart_name: str = REQ(),
-    min_length: Optional[int] = REQ(converter=to_non_negative_int, default=None),
-    start: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
-    end: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
-    realm: Optional[Realm] = None,
-    for_installation: bool = False,
-    remote: bool = False,
-    remote_realm_id: Optional[int] = None,
-    server: Optional["RemoteZulipServer"] = None,
-) -> HttpResponse:
-    TableType = Union[
-        Type["RemoteInstallationCount"],
-        Type[InstallationCount],
-        Type["RemoteRealmCount"],
-        Type[RealmCount],
-    ]
-    if for_installation:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table: TableType = RemoteInstallationCount
-            assert server is not None
-        else:
-            aggregate_table = InstallationCount
-    else:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table = RemoteRealmCount
-            assert server is not None
-            assert remote_realm_id is not None
-        else:
-            aggregate_table = RealmCount
-
-    tables: Union[Tuple[TableType], Tuple[TableType, Type[UserCount]]]
-
-    if chart_name == "number_of_humans":
-        stats = [
-            COUNT_STATS["1day_actives::day"],
-            COUNT_STATS["realm_active_humans::day"],
-            COUNT_STATS["active_users_audit:is_bot:day"],
-        ]
-        tables = (aggregate_table,)
-        subgroup_to_label: Dict[CountStat, Dict[Optional[str], str]] = {
-            stats[0]: {None: "_1day"},
-            stats[1]: {None: "_15day"},
-            stats[2]: {"false": "all_time"},
-        }
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_over_time":
-        stats = [COUNT_STATS["messages_sent:is_bot:hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_message_type":
-        stats = [COUNT_STATS["messages_sent:message_type:day"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {
-            stats[0]: {
-                "public_stream": _("Public streams"),
-                "private_stream": _("Private streams"),
-                "private_message": _("Direct messages"),
-                "huddle_message": _("Group direct messages"),
-            }
-        }
-        labels_sort_function = lambda data: sort_by_totals(data["everyone"])
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_client":
-        stats = [COUNT_STATS["messages_sent:client:day"]]
-        tables = (aggregate_table, UserCount)
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {
-            stats[0]: {str(id): name for id, name in Client.objects.values_list("id", "name")}
-        }
-        labels_sort_function = sort_client_labels
-        include_empty_subgroups = False
-    elif chart_name == "messages_read_over_time":
-        stats = [COUNT_STATS["messages_read::hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {None: "read"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    else:
-        raise JsonableError(_("Unknown chart name: {}").format(chart_name))
-
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None:
-        start = convert_to_UTC(start)
-    if end is not None:
-        end = convert_to_UTC(end)
-    if start is not None and end is not None and start > end:
-        raise JsonableError(
-            _("Start time is later than end time. Start: {start}, End: {end}").format(
-                start=start,
-                end=end,
-            )
-        )
-
-    if realm is None:
-        # Note that this value is invalid for Remote tables; be
-        # careful not to access it in those code paths.
-        realm = user_profile.realm
-
-    if remote:
-        # For remote servers, we don't have fillstate data, and thus
-        # should simply use the first and last data points for the
-        # table.
-        assert server is not None
-        assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
-        aggregate_table_remote = cast(
-            Union[Type[RemoteInstallationCount], Type[RemoteRealmCount]], aggregate_table
-        )  # https://stackoverflow.com/questions/68540528/mypy-assertions-on-the-types-of-types
-        if not aggregate_table_remote.objects.filter(server=server).exists():
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-        if start is None:
-            first = aggregate_table_remote.objects.filter(server=server).first()
-            assert first is not None
-            start = first.end_time
-        if end is None:
-            last = aggregate_table_remote.objects.filter(server=server).last()
-            assert last is not None
-            end = last.end_time
-    else:
-        # Otherwise, we can use tables on the current server to
-        # determine a nice range, and some additional validation.
-        if start is None:
-            if for_installation:
-                start = installation_epoch()
-            else:
-                start = realm.date_created
-        if end is None:
-            end = max(
-                stat.last_successful_fill() or datetime.min.replace(tzinfo=timezone.utc)
-                for stat in stats
-            )
-
-        if start > end and (timezone_now() - start > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION):
-            logging.warning(
-                "User from realm %s attempted to access /stats, but the computed "
-                "start time: %s (creation of realm or installation) is later than the computed "
-                "end time: %s (last successful analytics update). Is the "
-                "analytics cron job running?",
-                realm.string_id,
-                start,
-                end,
-            )
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-
-    assert len({stat.frequency for stat in stats}) == 1
-    end_times = time_range(start, end, stats[0].frequency, min_length)
-    data: Dict[str, Any] = {
-        "end_times": [int(end_time.timestamp()) for end_time in end_times],
-        "frequency": stats[0].frequency,
-    }
-
-    aggregation_level = {
-        InstallationCount: "everyone",
-        RealmCount: "everyone",
-        UserCount: "user",
-    }
-    if settings.ZILENCER_ENABLED:
-        aggregation_level[RemoteInstallationCount] = "everyone"
-        aggregation_level[RemoteRealmCount] = "everyone"
-
-    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
-    id_value = {
-        InstallationCount: -1,
-        RealmCount: realm.id,
-        UserCount: user_profile.id,
-    }
-    if settings.ZILENCER_ENABLED:
-        if server is not None:
-            id_value[RemoteInstallationCount] = server.id
-        # TODO: RemoteRealmCount logic doesn't correctly handle
-        # filtering by server_id as well.
-        if remote_realm_id is not None:
-            id_value[RemoteRealmCount] = remote_realm_id
-
-    for table in tables:
-        data[aggregation_level[table]] = {}
-        for stat in stats:
-            data[aggregation_level[table]].update(
-                get_time_series_by_subgroup(
-                    stat,
-                    table,
-                    id_value[table],
-                    end_times,
-                    subgroup_to_label[stat],
-                    include_empty_subgroups,
-                )
-            )
-
-    if labels_sort_function is not None:
-        data["display_order"] = labels_sort_function(data)
-    else:
-        data["display_order"] = None
-    return json_success(request, data=data)
-
-
-def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
-    totals = [(sum(values), label) for label, values in value_arrays.items()]
-    totals.sort(reverse=True)
-    return [label for total, label in totals]
-
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data: Dict[str, Dict[str, List[int]]]) -> List[str]:
-    realm_order = sort_by_totals(data["everyone"])
-    user_order = sort_by_totals(data["user"])
-    label_sort_values: Dict[str, float] = {}
-    for i, label in enumerate(realm_order):
-        label_sort_values[label] = i
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
-
-
-CountT = TypeVar("CountT", bound=BaseCount)
-
-
-def table_filtered_to_id(table: Type[CountT], key_id: int) -> QuerySet[CountT]:
-    if table == RealmCount:
-        return table.objects.filter(realm_id=key_id)
-    elif table == UserCount:
-        return table.objects.filter(user_id=key_id)
-    elif table == StreamCount:
-        return table.objects.filter(stream_id=key_id)
-    elif table == InstallationCount:
-        return table.objects.all()
-    elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
-        return table.objects.filter(server_id=key_id)
-    elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
-        return table.objects.filter(realm_id=key_id)
-    else:
-        raise AssertionError(f"Unknown table: {table}")
-
-
-def client_label_map(name: str) -> str:
-    if name == "website":
-        return "Web app"
-    if name.startswith("desktop app"):
-        return "Old desktop app"
-    if name == "ZulipElectron":
-        return "Desktop app"
-    if name == "ZulipTerminal":
-        return "Terminal app"
-    if name == "ZulipAndroid":
-        return "Old Android app"
-    if name == "ZulipiOS":
-        return "Old iOS app"
-    if name == "ZulipMobile":
-        return "Mobile app"
-    if name in ["ZulipPython", "API: Python"]:
-        return "Python API"
-    if name.startswith("Zulip") and name.endswith("Webhook"):
-        return name[len("Zulip") : -len("Webhook")] + " webhook"
-    return name
-
-
-def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[int]]:
-    mapped_arrays: Dict[str, List[int]] = {}
-    for label, array in value_arrays.items():
-        mapped_label = client_label_map(label)
-        if mapped_label in mapped_arrays:
-            for i in range(len(array)):
-                mapped_arrays[mapped_label][i] += value_arrays[label][i]
-        else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
-    return mapped_arrays
-
-
-def get_time_series_by_subgroup(
-    stat: CountStat,
-    table: Type[BaseCount],
-    key_id: int,
-    end_times: List[datetime],
-    subgroup_to_label: Dict[Optional[str], str],
-    include_empty_subgroups: bool,
-) -> Dict[str, List[int]]:
-    queryset = (
-        table_filtered_to_id(table, key_id)
-        .filter(property=stat.property)
-        .values_list("subgroup", "end_time", "value")
-    )
-    value_dicts: Dict[Optional[str], Dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
-    for subgroup, end_time, value in queryset:
-        value_dicts[subgroup][end_time] = value
-    value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
-        if (subgroup in value_dicts) or include_empty_subgroups:
-            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
-
-    if stat == COUNT_STATS["messages_sent:client:day"]:
-        # HACK: We rewrite these arrays to collapse the Client objects
-        # with similar names into a single sum, and generally give
-        # them better names
-        return rewrite_client_arrays(value_arrays)
-    return value_arrays

analytics/views/support.py

@@ -1,408 +0,0 @@
-import urllib
-from contextlib import suppress
-from dataclasses import dataclass
-from datetime import timedelta
-from decimal import Decimal
-from typing import Any, Dict, Iterable, List, Optional
-from urllib.parse import urlencode
-
-from django.conf import settings
-from django.core.exceptions import ValidationError
-from django.core.validators import URLValidator
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
-from django.shortcuts import render
-from django.urls import reverse
-from django.utils.timesince import timesince
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-
-from confirmation.models import Confirmation, confirmation_url
-from confirmation.settings import STATUS_USED
-from zerver.actions.create_realm import do_change_realm_subdomain
-from zerver.actions.realm_settings import (
-    do_change_realm_org_type,
-    do_change_realm_plan_type,
-    do_deactivate_realm,
-    do_scrub_realm,
-    do_send_realm_reactivation_email,
-)
-from zerver.actions.users import do_delete_user_preserving_messages
-from zerver.decorator import require_server_admin
-from zerver.forms import check_subdomain_available
-from zerver.lib.exceptions import JsonableError
-from zerver.lib.realm_icon import realm_icon_url
-from zerver.lib.request import REQ, has_request_variables
-from zerver.lib.subdomains import get_subdomain_from_hostname
-from zerver.lib.validator import check_bool, check_string_in, to_decimal, to_non_negative_int
-from zerver.models import (
-    MultiuseInvite,
-    PreregistrationRealm,
-    PreregistrationUser,
-    Realm,
-    RealmReactivationStatus,
-    UserProfile,
-    get_org_type_display_name,
-    get_realm,
-    get_user_profile_by_id,
-)
-from zerver.views.invite import get_invitee_emails_set
-
-if settings.BILLING_ENABLED:
-    from corporate.lib.stripe import approve_sponsorship as do_approve_sponsorship
-    from corporate.lib.stripe import (
-        attach_discount_to_realm,
-        downgrade_at_the_end_of_billing_cycle,
-        downgrade_now_without_creating_additional_invoices,
-        get_discount_for_realm,
-        get_latest_seat_count,
-        make_end_of_cycle_updates_if_needed,
-        switch_realm_from_standard_to_plus_plan,
-        update_billing_method_of_current_plan,
-        update_sponsorship_status,
-        void_all_open_invoices,
-    )
-    from corporate.models import (
-        Customer,
-        CustomerPlan,
-        get_current_plan_by_realm,
-        get_customer_by_realm,
-    )
-
-
-def get_plan_name(plan_type: int) -> str:
-    return {
-        Realm.PLAN_TYPE_SELF_HOSTED: "self-hosted",
-        Realm.PLAN_TYPE_LIMITED: "limited",
-        Realm.PLAN_TYPE_STANDARD: "standard",
-        Realm.PLAN_TYPE_STANDARD_FREE: "open source",
-        Realm.PLAN_TYPE_PLUS: "plus",
-    }[plan_type]
-
-
-def get_confirmations(
-    types: List[int], object_ids: Iterable[int], hostname: Optional[str] = None
-) -> List[Dict[str, Any]]:
-    lowest_datetime = timezone_now() - timedelta(days=30)
-    confirmations = Confirmation.objects.filter(
-        type__in=types, object_id__in=object_ids, date_sent__gte=lowest_datetime
-    )
-    confirmation_dicts = []
-    for confirmation in confirmations:
-        realm = confirmation.realm
-        content_object = confirmation.content_object
-
-        type = confirmation.type
-        expiry_date = confirmation.expiry_date
-
-        assert content_object is not None
-        if hasattr(content_object, "status"):
-            if content_object.status == STATUS_USED:
-                link_status = "Link has been used"
-            else:
-                link_status = "Link has not been used"
-        else:
-            link_status = ""
-
-        now = timezone_now()
-        if expiry_date is None:
-            expires_in = "Never"
-        elif now < expiry_date:
-            expires_in = timesince(now, expiry_date)
-        else:
-            expires_in = "Expired"
-
-        url = confirmation_url(confirmation.confirmation_key, realm, type)
-        confirmation_dicts.append(
-            {
-                "object": confirmation.content_object,
-                "url": url,
-                "type": type,
-                "link_status": link_status,
-                "expires_in": expires_in,
-            }
-        )
-    return confirmation_dicts
-
-
-VALID_MODIFY_PLAN_METHODS = [
-    "downgrade_at_billing_cycle_end",
-    "downgrade_now_without_additional_licenses",
-    "downgrade_now_void_open_invoices",
-    "upgrade_to_plus",
-]
-
-VALID_STATUS_VALUES = [
-    "active",
-    "deactivated",
-]
-
-VALID_BILLING_METHODS = [
-    "send_invoice",
-    "charge_automatically",
-]
-
-
-@dataclass
-class PlanData:
-    customer: Optional["Customer"] = None
-    current_plan: Optional["CustomerPlan"] = None
-    licenses: Optional[int] = None
-    licenses_used: Optional[int] = None
-
-
-@require_server_admin
-@has_request_variables
-def support(
-    request: HttpRequest,
-    realm_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-    plan_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-    discount: Optional[Decimal] = REQ(default=None, converter=to_decimal),
-    new_subdomain: Optional[str] = REQ(default=None),
-    status: Optional[str] = REQ(default=None, str_validator=check_string_in(VALID_STATUS_VALUES)),
-    billing_method: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_BILLING_METHODS)
-    ),
-    sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
-    approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
-    modify_plan: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
-    ),
-    scrub_realm: bool = REQ(default=False, json_validator=check_bool),
-    delete_user_by_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-    query: Optional[str] = REQ("q", default=None),
-    org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-) -> HttpResponse:
-    context: Dict[str, Any] = {}
-
-    if "success_message" in request.session:
-        context["success_message"] = request.session["success_message"]
-        del request.session["success_message"]
-
-    if settings.BILLING_ENABLED and request.method == "POST":
-        # We check that request.POST only has two keys in it: The
-        # realm_id and a field to change.
-        keys = set(request.POST.keys())
-        if "csrfmiddlewaretoken" in keys:
-            keys.remove("csrfmiddlewaretoken")
-        if len(keys) != 2:
-            raise JsonableError(_("Invalid parameters"))
-
-        assert realm_id is not None
-        realm = Realm.objects.get(id=realm_id)
-
-        acting_user = request.user
-        assert isinstance(acting_user, UserProfile)
-        if plan_type is not None:
-            current_plan_type = realm.plan_type
-            do_change_realm_plan_type(realm, plan_type, acting_user=acting_user)
-            msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(plan_type)} "
-            context["success_message"] = msg
-        elif org_type is not None:
-            current_realm_type = realm.org_type
-            do_change_realm_org_type(realm, org_type, acting_user=acting_user)
-            msg = f"Org type of {realm.string_id} changed from {get_org_type_display_name(current_realm_type)} to {get_org_type_display_name(org_type)} "
-            context["success_message"] = msg
-        elif discount is not None:
-            current_discount = get_discount_for_realm(realm) or 0
-            attach_discount_to_realm(realm, discount, acting_user=acting_user)
-            context[
-                "success_message"
-            ] = f"Discount of {realm.string_id} changed to {discount}% from {current_discount}%."
-        elif new_subdomain is not None:
-            old_subdomain = realm.string_id
-            try:
-                check_subdomain_available(new_subdomain)
-            except ValidationError as error:
-                context["error_message"] = error.message
-            else:
-                do_change_realm_subdomain(realm, new_subdomain, acting_user=acting_user)
-                request.session[
-                    "success_message"
-                ] = f"Subdomain changed from {old_subdomain} to {new_subdomain}"
-                return HttpResponseRedirect(
-                    reverse("support") + "?" + urlencode({"q": new_subdomain})
-                )
-        elif status is not None:
-            if status == "active":
-                do_send_realm_reactivation_email(realm, acting_user=acting_user)
-                context[
-                    "success_message"
-                ] = f"Realm reactivation email sent to admins of {realm.string_id}."
-            elif status == "deactivated":
-                do_deactivate_realm(realm, acting_user=acting_user)
-                context["success_message"] = f"{realm.string_id} deactivated."
-        elif billing_method is not None:
-            if billing_method == "send_invoice":
-                update_billing_method_of_current_plan(
-                    realm, charge_automatically=False, acting_user=acting_user
-                )
-                context[
-                    "success_message"
-                ] = f"Billing method of {realm.string_id} updated to pay by invoice."
-            elif billing_method == "charge_automatically":
-                update_billing_method_of_current_plan(
-                    realm, charge_automatically=True, acting_user=acting_user
-                )
-                context[
-                    "success_message"
-                ] = f"Billing method of {realm.string_id} updated to charge automatically."
-        elif sponsorship_pending is not None:
-            if sponsorship_pending:
-                update_sponsorship_status(realm, True, acting_user=acting_user)
-                context["success_message"] = f"{realm.string_id} marked as pending sponsorship."
-            else:
-                update_sponsorship_status(realm, False, acting_user=acting_user)
-                context["success_message"] = f"{realm.string_id} is no longer pending sponsorship."
-        elif approve_sponsorship:
-            do_approve_sponsorship(realm, acting_user=acting_user)
-            context["success_message"] = f"Sponsorship approved for {realm.string_id}"
-        elif modify_plan is not None:
-            if modify_plan == "downgrade_at_billing_cycle_end":
-                downgrade_at_the_end_of_billing_cycle(realm)
-                context[
-                    "success_message"
-                ] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
-            elif modify_plan == "downgrade_now_without_additional_licenses":
-                downgrade_now_without_creating_additional_invoices(realm)
-                context[
-                    "success_message"
-                ] = f"{realm.string_id} downgraded without creating additional invoices"
-            elif modify_plan == "downgrade_now_void_open_invoices":
-                downgrade_now_without_creating_additional_invoices(realm)
-                voided_invoices_count = void_all_open_invoices(realm)
-                context[
-                    "success_message"
-                ] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
-            elif modify_plan == "upgrade_to_plus":
-                switch_realm_from_standard_to_plus_plan(realm)
-                context["success_message"] = f"{realm.string_id} upgraded to Plus"
-        elif scrub_realm:
-            do_scrub_realm(realm, acting_user=acting_user)
-            context["success_message"] = f"{realm.string_id} scrubbed."
-        elif delete_user_by_id:
-            user_profile_for_deletion = get_user_profile_by_id(delete_user_by_id)
-            user_email = user_profile_for_deletion.delivery_email
-            assert user_profile_for_deletion.realm == realm
-            do_delete_user_preserving_messages(user_profile_for_deletion)
-            context["success_message"] = f"{user_email} in {realm.subdomain} deleted."
-
-    if query:
-        key_words = get_invitee_emails_set(query)
-
-        case_insensitive_users_q = Q()
-        for key_word in key_words:
-            case_insensitive_users_q |= Q(delivery_email__iexact=key_word)
-        users = set(UserProfile.objects.filter(case_insensitive_users_q))
-        realms = set(Realm.objects.filter(string_id__in=key_words))
-
-        for key_word in key_words:
-            try:
-                URLValidator()(key_word)
-                parse_result = urllib.parse.urlparse(key_word)
-                hostname = parse_result.hostname
-                assert hostname is not None
-                if parse_result.port:
-                    hostname = f"{hostname}:{parse_result.port}"
-                subdomain = get_subdomain_from_hostname(hostname)
-                with suppress(Realm.DoesNotExist):
-                    realms.add(get_realm(subdomain))
-            except ValidationError:
-                users.update(UserProfile.objects.filter(full_name__iexact=key_word))
-
-        # full_names can have , in them
-        users.update(UserProfile.objects.filter(full_name__iexact=query))
-
-        context["users"] = users
-        context["realms"] = realms
-
-        confirmations: List[Dict[str, Any]] = []
-
-        preregistration_user_ids = [
-            user.id for user in PreregistrationUser.objects.filter(email__in=key_words)
-        ]
-        confirmations += get_confirmations(
-            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION],
-            preregistration_user_ids,
-            hostname=request.get_host(),
-        )
-
-        preregistration_realm_ids = [
-            user.id for user in PreregistrationRealm.objects.filter(email__in=key_words)
-        ]
-        confirmations += get_confirmations(
-            [Confirmation.REALM_CREATION],
-            preregistration_realm_ids,
-            hostname=request.get_host(),
-        )
-
-        multiuse_invite_ids = [
-            invite.id for invite in MultiuseInvite.objects.filter(realm__in=realms)
-        ]
-        confirmations += get_confirmations([Confirmation.MULTIUSE_INVITE], multiuse_invite_ids)
-
-        realm_reactivation_status_objects = RealmReactivationStatus.objects.filter(realm__in=realms)
-        confirmations += get_confirmations(
-            [Confirmation.REALM_REACTIVATION], [obj.id for obj in realm_reactivation_status_objects]
-        )
-
-        context["confirmations"] = confirmations
-
-        # We want a union of all realms that might appear in the search result,
-        # but not necessary as a separate result item.
-        # Therefore, we do not modify the realms object in the context.
-        all_realms = realms.union(
-            [
-                confirmation["object"].realm
-                for confirmation in confirmations
-                # For confirmations, we only display realm details when the type is USER_REGISTRATION
-                # or INVITATION.
-                if confirmation["type"] in (Confirmation.USER_REGISTRATION, Confirmation.INVITATION)
-            ]
-            + [user.realm for user in users]
-        )
-        plan_data: Dict[int, PlanData] = {}
-        for realm in all_realms:
-            current_plan = get_current_plan_by_realm(realm)
-            plan_data[realm.id] = PlanData(
-                customer=get_customer_by_realm(realm),
-                current_plan=current_plan,
-            )
-            if current_plan is not None:
-                new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(
-                    current_plan, timezone_now()
-                )
-                if last_ledger_entry is not None:
-                    if new_plan is not None:
-                        plan_data[realm.id].current_plan = new_plan
-                    else:
-                        plan_data[realm.id].current_plan = current_plan
-                    plan_data[realm.id].licenses = last_ledger_entry.licenses
-                    plan_data[realm.id].licenses_used = get_latest_seat_count(realm)
-        context["plan_data"] = plan_data
-
-    def get_realm_owner_emails_as_string(realm: Realm) -> str:
-        return ", ".join(
-            realm.get_human_owner_users()
-            .order_by("delivery_email")
-            .values_list("delivery_email", flat=True)
-        )
-
-    def get_realm_admin_emails_as_string(realm: Realm) -> str:
-        return ", ".join(
-            realm.get_human_admin_users(include_realm_owners=False)
-            .order_by("delivery_email")
-            .values_list("delivery_email", flat=True)
-        )
-
-    context["get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string
-    context["get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string
-    context["get_discount_for_realm"] = get_discount_for_realm
-    context["get_org_type_display_name"] = get_org_type_display_name
-    context["realm_icon_url"] = realm_icon_url
-    context["Confirmation"] = Confirmation
-    context["sorted_realm_types"] = sorted(
-        Realm.ORG_TYPES.values(), key=lambda d: d["display_order"]
-    )
-
-    return render(request, "analytics/support.html", context=context)

analytics/views/user_activity.py

@@ -1,106 +0,0 @@
-from typing import Any, Dict, List, Tuple
-
-from django.conf import settings
-from django.db.models import QuerySet
-from django.http import HttpRequest, HttpResponse
-from django.shortcuts import render
-
-from analytics.views.activity_common import (
-    format_date_for_activity_reports,
-    get_user_activity_summary,
-    make_table,
-)
-from zerver.decorator import require_server_admin
-from zerver.models import UserActivity, UserProfile, get_user_profile_by_id
-
-if settings.BILLING_ENABLED:
-    pass
-
-
-def get_user_activity_records(
-    user_profile: UserProfile,
-) -> QuerySet[UserActivity]:
-    fields = [
-        "user_profile__full_name",
-        "query",
-        "client__name",
-        "count",
-        "last_visit",
-    ]
-
-    records = UserActivity.objects.filter(
-        user_profile=user_profile,
-    )
-    records = records.order_by("-last_visit")
-    records = records.select_related("user_profile", "client").only(*fields)
-    return records
-
-
-def raw_user_activity_table(records: QuerySet[UserActivity]) -> str:
-    cols = [
-        "query",
-        "client",
-        "count",
-        "last_visit",
-    ]
-
-    def row(record: UserActivity) -> List[Any]:
-        return [
-            record.query,
-            record.client.name,
-            record.count,
-            format_date_for_activity_reports(record.last_visit),
-        ]
-
-    rows = list(map(row, records))
-    title = "Raw data"
-    return make_table(title, cols, rows)
-
-
-def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
-    rows = []
-    for k, v in user_summary.items():
-        if k in ("name", "user_profile_id"):
-            continue
-        client = k
-        count = v["count"]
-        last_visit = v["last_visit"]
-        row = [
-            format_date_for_activity_reports(last_visit),
-            client,
-            count,
-        ]
-        rows.append(row)
-
-    rows = sorted(rows, key=lambda r: r[0], reverse=True)
-
-    cols = [
-        "last_visit",
-        "client",
-        "count",
-    ]
-
-    title = "User activity"
-    return make_table(title, cols, rows)
-
-
-@require_server_admin
-def get_user_activity(request: HttpRequest, user_profile_id: int) -> HttpResponse:
-    user_profile = get_user_profile_by_id(user_profile_id)
-    records = get_user_activity_records(user_profile)
-
-    data: List[Tuple[str, str]] = []
-    user_summary = get_user_activity_summary(records)
-    content = user_activity_summary_table(user_summary)
-
-    data += [("Summary", content)]
-
-    content = raw_user_activity_table(records)
-    data += [("Info", content)]
-
-    title = user_profile.delivery_email
-    return render(
-        request,
-        "analytics/activity.html",
-        context=dict(data=data, title=title),
-    )

api/MANIFEST.in

@@ -0,0 +1,11 @@
+recursive-include integrations *
+include README.md
+include examples/zuliprc
+include examples/send-message
+include examples/subscribe
+include examples/get-public-streams
+include examples/unsubscribe
+include examples/list-members
+include examples/list-subscriptions
+include examples/print-messages
+include examples/recent-messages

api/README.md

@@ -0,0 +1,159 @@
+#### Dependencies
+
+The [Zulip API](https://zulip.com/api) Python bindings require the
+following Python libraries:
+
+* simplejson
+* requests (version >= 0.12.1)
+
+
+#### Installing
+
+This package uses distutils, so you can just run:
+
+    python setup.py install
+
+#### Using the API
+
+For now, the only fully supported API operation is sending a message.
+The other API queries work, but are under active development, so
+please make sure we know you're using them so that we can notify you
+as we make any changes to them.
+
+The easiest way to use these API bindings is to base your tools off
+of the example tools under examples/ in this distribution.
+
+If you place your API key in the config file `~/.zuliprc` the Python
+API bindings will automatically read it in. The format of the config
+file is as follows:
+
+    [api]
+    key=<api key from the web interface>
+    email=<your email address>
+    site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    site=https://api.zulip.com
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
+
+Alternatively, you may explicitly use "--user" and "--api-key" in our
+examples, which is especially useful if you are running several bots
+which share a home directory.
+
+The command line equivalents for other configuration options are:
+
+    --site=<your Zulip server's URI>
+    --insecure
+    --cert-bundle=<file>
+
+You can obtain your Zulip API key, create bots, and manage bots all
+from your Zulip [settings page](https://zulip.com/#settings).
+
+A typical simple bot sending API messages will look as follows:
+
+At the top of the file:
+
+    # Make sure the Zulip API distribution's root directory is in sys.path, then:
+    import zulip
+    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
+
+When you want to send a message:
+
+    message = {
+      "type": "stream",
+      "to": ["support"],
+      "subject": "your subject",
+      "content": "your content",
+    }
+    zulip_client.send_message(message)
+
+Additional examples:
+
+    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
+                         'subject': 'feedback', 'to': ['support']})
+    client.send_message({'type': 'private', 'content': 'Zulip rules!',
+                         'to': ['user1@example.com', 'user2@example.com']})
+
+send_message() returns a dict guaranteed to contain the following
+keys: msg, result.  For successful calls, result will be "success" and
+msg will be the empty string.  On error, result will be "error" and
+msg will describe what went wrong.
+
+#### Logging
+The Zulip API comes with a ZulipStream class which can be used with the
+logging module:
+
+```
+import zulip
+import logging
+stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
+logger = logging.getLogger("your_logger")
+logger.addHandler(logging.StreamHandler(stream))
+logger.setLevel(logging.DEBUG)
+logger.info("This is an INFO test.")
+logger.debug("This is a DEBUG test.")
+logger.warn("This is a WARN test.")
+logger.error("This is a ERROR test.")
+```
+
+#### Sending messages
+
+You can use the included `zulip-send` script to send messages via the
+API directly from existing scripts.
+
+    zulip-send hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+Alternatively, if you don't want to use your ~/.zuliprc file:
+
+    zulip-send --user shakespeare-bot@example.com \
+        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+        hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.

api/bin/zulip-send

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# zulip-send -- Sends a message to the specified recipients.
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import os
+import optparse
+import logging
+
+
+logging.basicConfig()
+
+log = logging.getLogger('zulip-send')
+
+def do_send_message(client, message_data ):
+    '''Sends a message and optionally prints status about the same.'''
+
+    if message_data['type'] == 'stream':
+        log.info('Sending message to stream "%s", subject "%s"... ' % \
+            (message_data['to'], message_data['subject']))
+    else:
+        log.info('Sending message to %s... ' % message_data['to'])
+    response = client.send_message(message_data)
+    if response['result'] == 'success':
+        log.info('Message sent.')
+        return True
+    else:
+        log.error(response['msg'])
+        return False
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    usage = """%prog [options] [recipient...]
+
+    Sends a message specified recipients.
+
+    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
+              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
+
+    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
+    '--user' and '--api-key' arguments.
+    """
+
+    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+    import zulip
+
+    parser = optparse.OptionParser(usage=usage)
+
+    # Grab parser options from the API common set
+    parser.add_option_group(zulip.generate_option_group(parser))
+
+    parser.add_option('-m', '--message',
+                      help='Specifies the message to send, prevents interactive prompting.')
+
+    group = optparse.OptionGroup(parser, 'Stream parameters')
+    group.add_option('-s', '--stream',
+                      dest='stream',
+                      action='store',
+                      help='Allows the user to specify a stream for the message.')
+    group.add_option('-S', '--subject',
+                      dest='subject',
+                      action='store',
+                      help='Allows the user to specify a subject for the message.')
+    parser.add_option_group(group)
+
+
+    (options, recipients) = parser.parse_args(argv[1:])
+
+    if options.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+    # Sanity check user data
+    if len(recipients) != 0 and (options.stream or options.subject):
+        parser.error('You cannot specify both a username and a stream/subject.')
+    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
+        parser.error('Stream messages must have a subject')
+    if len(recipients) == 0 and not (options.stream and options.subject):
+        parser.error('You must specify a stream/subject or at least one recipient.')
+
+    client = zulip.init_from_options(options)
+
+    if not options.message:
+        options.message = sys.stdin.read()
+
+    if options.stream:
+        message_data = {
+            'type': 'stream',
+            'content': options.message,
+            'subject': options.subject,
+            'to': options.stream,
+        }
+    else:
+        message_data = {
+            'type': 'private',
+            'content': options.message,
+            'to': recipients,
+        }
+
+    if not do_send_message(client, message_data):
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

api/examples/create-user

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+from os import path
+import optparse
+
+usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
+
+Create a user. You must be a realm admin to use this API, and the user
+will be created in your realm.
+
+Example: create-user --site=http://localhost:9991 --user=rwbarton@zulip.com --new-email=jarthur@zulip.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
+"""
+
+sys.path.append(path.join(path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--new-email')
+parser.add_option('--new-password')
+parser.add_option('--new-full-name')
+parser.add_option('--new-short-name')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.create_user({
+        'email': options.new_email,
+        'password': options.new_password,
+        'full_name': options.new_full_name,
+        'short_name': options.new_short_name
+        }))

api/examples/edit-message

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
+
+Edits a message that you sent
+
+Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--message-id', default="")
+parser.add_option('--subject', default="")
+parser.add_option('--content',   default="")
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "message_id": options.message_id,
+}
+if options.subject != "":
+    message_data["subject"] = options.subject
+if options.content != "":
+    message_data["content"] = options.content
+print(client.update_message(message_data))

api/examples/get-public-streams

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out all the public streams in the realm.
+
+Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_streams(include_public=True, include_subscribed=False))

api/examples/list-members

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
+
+List the names and e-mail addresses of the people in your realm.
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+for user in client.get_members()["members"]:
+    print(user["full_name"], user["email"])

api/examples/list-subscriptions

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out a list of the user's subscriptions.
+
+Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.list_subscriptions())

api/examples/print-events

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out certain events received by the indicated bot or user matching the filter below.
+
+Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_event(event):
+    print(event)
+
+# This is a blocking call, and will continuously poll for new events
+# Note also the filter here is messages to the stream Denmark; if you
+# don't specify event_types it'll print all events.
+client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])

api/examples/print-messages

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out each message received by the indicated bot or user.
+
+Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_message(message):
+    print(message)
+
+# This is a blocking call, and will continuously poll for new messages
+client.call_on_each_message(print_message)

api/examples/print-next-message

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """print-next-message --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out the next message received by the user.
+
+Example: print-next-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_messages({}))

api/examples/recent-messages

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages recieved by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+req = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.do_api_query(req, zulip.API_VERSTRING + 'messages', method='GET')
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])

api/examples/send-message

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
+
+Sends a test message to the specified recipients.
+
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--subject', default="test")
+parser.add_option('--message', default="test message")
+parser.add_option('--type',   default='private')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if len(args) == 0:
+    parser.error("You must specify recipients")
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "type": options.type,
+    "content": options.message,
+    "subject": options.subject,
+    "to": args,
+}
+print(client.send_message(message_data))

api/examples/subscribe

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is subscribed to the listed streams.
+
+Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))

api/examples/unsubscribe

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is not subscribed to the listed streams.
+
+Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.remove_subscriptions(options.streams.split()))

api/examples/zuliprc

@@ -0,0 +1,4 @@
+; Save this file as ~/.zuliprc
+[api]
+key=<your bot's api key from the web interface>
+email=<your bot's email address>

api/integrations/asana/zulip_asana_config.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+### REQUIRED CONFIGURATION ###
+
+# Change these values to your Asana credentials.
+ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# Change these values to the credentials for your Asana bot.
+ZULIP_USER = "asana-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The Zulip stream that will receive Asana task updates.
+ZULIP_STREAM_NAME = "asana"
+
+
+### OPTIONAL CONFIGURATION ###
+
+# Set to None for logging to stdout when testing, and to a file for
+# logging in production.
+#LOG_FILE = "/var/tmp/zulip_asana.log"
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_asana.state"
+
+# When initially started, how many hours of messages to include.
+ASANA_INITIAL_HISTORY_HOURS = 1
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://api.zulip.com"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None