Release Zulip Server 4.7

Zulip attempts to validate that the regular expressions that admins
enter for linkifiers are well-formatted, and only contain a specific
subset of regex grammar.  The process of checking these
properties (via a regex!) can cause denial-of-service via
backtracking.

Furthermore, this validation itself does not prevent the creation of
linkifiers which themselves cause denial-of-service when they are
executed.  As the validator accepts literally anything inside of a
`(?P<word>...)` block, any quadratic backtracking expression can be
hidden therein.

Switch user-provided linkifier patterns to be matched in the Markdown
processor by the `re2` library, which is guaranteed constant-time.
This somewhat limits the possible features of the regular
expression (notably, look-head and -behind, and back-references);
however, these features had never been advertised as working in the
context of linkifiers.

A migration removes any existing linkifiers which would not function
under re2, after printing them for posterity during the upgrade; they
are unlikely to be common, and are impossible to fix automatically.

The denial-of-service in the linkifier validator was discovered by
@erik-krogh and @yoff, as GHSL-2021-118.

.browserslistrc

@@ -0,0 +1,5 @@
+> 0.15%
+> 0.15% in US
+last 2 versions
+Firefox ESR
+not dead

.codespellignore

@@ -1,31 +0,0 @@
-te
-ans
-pullrequest
-ist
-cros
-wit
-nwe
-circularly
-ned
-ba
-ressemble
-ser
-sur
-hel
-fpr
-alls
-nd
-ot
-womens
-vise
-falsy
-ro
-derails
-forin
-uper
-slac
-couldn
-ges
-assertIn
-thirdparty
-asend

.editorconfig

@@ -8,11 +8,10 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[[shell]]
-binary_next_line = true
-switch_case_indent = true
+binary_next_line = true  # for shfmt
+switch_case_indent = true  # for shfmt
 
-[{*.{cjs,cts,js,json,mjs,mts,ts},check-openapi}]
+[{*.{js,json,ts},check-openapi}]
 max_line_length = 100
 
 [*.{py,pyi}]

.eslintignore

@@ -0,0 +1,14 @@
+# This is intended for generated files and vendored third-party files.
+# For our source code, instead of adding files here, consider using
+# specific eslint-disable comments in the files themselves.
+
+/docs/_build
+/static/generated
+/static/third
+/static/webpack-bundles
+/var/*
+!/var/puppeteer
+/var/puppeteer/*
+!/var/puppeteer/test_credentials.d.ts
+/zulip-current-venv
+/zulip-py3-venv

.eslintrc.json

@@ -0,0 +1,230 @@
+{
+    "env": {
+        "es2020": true,
+        "node": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:import/errors",
+        "plugin:import/warnings",
+        "plugin:unicorn/recommended",
+        "prettier"
+    ],
+    "parser": "@babel/eslint-parser",
+    "parserOptions": {
+        "warnOnUnsupportedTypeScriptVersion": false,
+        "sourceType": "unambiguous"
+    },
+    "reportUnusedDisableDirectives": true,
+    "rules": {
+        "array-callback-return": "error",
+        "arrow-body-style": "error",
+        "block-scoped-var": "error",
+        "consistent-return": "error",
+        "curly": "error",
+        "dot-notation": "error",
+        "eqeqeq": "error",
+        "guard-for-in": "error",
+        "import/extensions": "error",
+        "import/first": "error",
+        "import/newline-after-import": "error",
+        "import/no-useless-path-segments": "error",
+        "import/order": [
+            "error",
+            {
+                "alphabetize": {"order": "asc"},
+                "newlines-between": "always"
+            }
+        ],
+        "import/unambiguous": "error",
+        "lines-around-directive": "error",
+        "new-cap": "error",
+        "no-alert": "error",
+        "no-array-constructor": "error",
+        "no-bitwise": "error",
+        "no-caller": "error",
+        "no-catch-shadow": "error",
+        "no-constant-condition": ["error", {"checkLoops": false}],
+        "no-div-regex": "error",
+        "no-duplicate-imports": "error",
+        "no-else-return": "error",
+        "no-eq-null": "error",
+        "no-eval": "error",
+        "no-implicit-coercion": "error",
+        "no-implied-eval": "error",
+        "no-inner-declarations": "off",
+        "no-iterator": "error",
+        "no-label-var": "error",
+        "no-labels": "error",
+        "no-loop-func": "error",
+        "no-multi-str": "error",
+        "no-native-reassign": "error",
+        "no-new-func": "error",
+        "no-new-object": "error",
+        "no-new-wrappers": "error",
+        "no-octal-escape": "error",
+        "no-plusplus": "error",
+        "no-proto": "error",
+        "no-return-assign": "error",
+        "no-script-url": "error",
+        "no-self-compare": "error",
+        "no-sync": "error",
+        "no-throw-literal": "error",
+        "no-undef-init": "error",
+        "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
+        "no-unused-expressions": "error",
+        "no-use-before-define": ["error", {"functions": false}],
+        "no-useless-concat": "error",
+        "no-useless-constructor": "error",
+        "no-var": "error",
+        "object-shorthand": "error",
+        "one-var": ["error", "never"],
+        "prefer-arrow-callback": "error",
+        "prefer-const": [
+            "error",
+            {
+                "ignoreReadBeforeAssign": true
+            }
+        ],
+        "radix": "error",
+        "sort-imports": ["error", {"ignoreDeclarationSort": true}],
+        "spaced-comment": ["error", "always", {"markers": ["/"]}],
+        "strict": "error",
+        "unicorn/consistent-function-scoping": "off",
+        "unicorn/explicit-length-check": "off",
+        "unicorn/filename-case": "off",
+        "unicorn/no-nested-ternary": "off",
+        "unicorn/no-null": "off",
+        "unicorn/no-process-exit": "off",
+        "unicorn/no-useless-undefined": "off",
+        "unicorn/number-literal-case": "off",
+        "unicorn/numeric-separators-style": "off",
+        "unicorn/prefer-module": "off",
+        "unicorn/prefer-node-protocol": "off",
+        "unicorn/prefer-spread": "off",
+        "unicorn/prefer-ternary": "off",
+        "unicorn/prevent-abbreviations": "off",
+        "valid-typeof": ["error", {"requireStringLiterals": true}],
+        "yoda": "error"
+    },
+    "overrides": [
+        {
+            "files": ["frontend_tests/puppeteer_lib/**", "frontend_tests/puppeteer_tests/**"],
+            "globals": {
+                "$": false,
+                "zulip_test": false
+            }
+        },
+        {
+            "files": ["static/js/**"],
+            "globals": {
+                "StripeCheckout": false
+            }
+        },
+        {
+            "files": ["**/*.ts"],
+            "extends": ["plugin:@typescript-eslint/recommended", "plugin:import/typescript"],
+            "parserOptions": {
+                "project": "tsconfig.json"
+            },
+            "rules": {
+                // Disable base rule to avoid conflict
+                "no-duplicate-imports": "off",
+                "no-unused-vars": "off",
+                "no-useless-constructor": "off",
+
+                "@typescript-eslint/array-type": "error",
+                "@typescript-eslint/await-thenable": "error",
+                "@typescript-eslint/consistent-type-assertions": "error",
+                "@typescript-eslint/consistent-type-imports": "error",
+                "@typescript-eslint/explicit-function-return-type": [
+                    "error",
+                    {"allowExpressions": true}
+                ],
+                "@typescript-eslint/member-ordering": "error",
+                "@typescript-eslint/no-duplicate-imports": "off",
+                "@typescript-eslint/no-explicit-any": "off",
+                "@typescript-eslint/no-extraneous-class": "error",
+                "@typescript-eslint/no-non-null-assertion": "off",
+                "@typescript-eslint/no-parameter-properties": "error",
+                "@typescript-eslint/no-unnecessary-qualifier": "error",
+                "@typescript-eslint/no-unnecessary-type-assertion": "error",
+                "@typescript-eslint/no-unused-vars": ["error", {"varsIgnorePattern": "^_"}],
+                "@typescript-eslint/no-use-before-define": "error",
+                "@typescript-eslint/no-useless-constructor": "error",
+                "@typescript-eslint/prefer-includes": "error",
+                "@typescript-eslint/prefer-regexp-exec": "error",
+                "@typescript-eslint/prefer-string-starts-ends-with": "error",
+                "@typescript-eslint/promise-function-async": "error",
+                "@typescript-eslint/unified-signatures": "error",
+                "no-undef": "error"
+            }
+        },
+        {
+            "files": ["**/*.d.ts"],
+            "rules": {
+                "import/unambiguous": "off"
+            }
+        },
+        {
+            "files": ["frontend_tests/**"],
+            "globals": {
+                "CSS": false,
+                "document": false,
+                "navigator": false,
+                "window": false
+            },
+            "rules": {
+                "no-sync": "off"
+            }
+        },
+        {
+            "files": ["tools/debug-require.js"],
+            "env": {
+                "browser": true,
+                "es2020": false
+            },
+            "rules": {
+                // Don’t require ES features that PhantomJS doesn’t support
+                // TODO: Toggle these settings now that we don't use PhantomJS
+                "no-var": "off",
+                "object-shorthand": "off",
+                "prefer-arrow-callback": "off"
+            }
+        },
+        {
+            "files": ["static/**"],
+            "env": {
+                "browser": true,
+                "node": false
+            },
+            "rules": {
+                "no-console": "error"
+            },
+            "settings": {
+                "import/resolver": "webpack"
+            }
+        },
+        {
+            "files": ["static/shared/**"],
+            "env": {
+                "browser": false,
+                "shared-node-browser": true
+            },
+            "rules": {
+                "import/no-restricted-paths": [
+                    "error",
+                    {
+                        "zones": [
+                            {
+                                "target": "./static/shared",
+                                "from": ".",
+                                "except": ["./node_modules", "./static/shared"]
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    ]
+}

.gitattributes

@@ -1,19 +1,4 @@
-# DIFFS: Noise suppression.
-#
-# Suppress noisy generated files in diffs.
-# (When you actually want to see these diffs, use `git diff -a`.)
-
-# Large test fixtures:
-corporate/tests/stripe_fixtures/*.json -diff
-
-
-# FORMATTING
-
-# Maintain LF (Unix-style) newlines in text files.
 *   text=auto eol=lf
-
-# Make sure various media files never get somehow auto-detected as text
-# and then newline-converted.
 *.gif binary
 *.jpg binary
 *.jpeg binary
@@ -26,7 +11,3 @@ corporate/tests/stripe_fixtures/*.json -diff
 *.otf binary
 *.tif binary
 *.ogg binary
-*.bson binary
-*.bmp binary
-*.mp3 binary
-*.pdf binary

.github/ISSUE_TEMPLATE/1_discussed_on_czo.md

@@ -1,10 +0,0 @@
----
-name: Issue discussed in the Zulip development community
-about: Bug report, feature or improvement already discussed on chat.zulip.org.
----
-
-<!-- Issue description -->
-
-<!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
-
-CZO thread

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -1,18 +0,0 @@
----
-name: Bug report
-about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
-labels: ["bug"]
----
-
-<!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
-
-<!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
-
-**Zulip Server and web app version:**
-
-- [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 9.x
-- [ ] Zulip Server 8.x
-- [ ] Zulip Server 7.x
-- [ ] Zulip Server 6.x or older
-- [ ] Other or not sure

.github/ISSUE_TEMPLATE/3_feature_request.md

@@ -1,6 +0,0 @@
----
-name: Feature or improvement request
-about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
----
-
-<!-- Describe the proposal, including how it would help you or your organization. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Possible bug
-    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
-    about: Report unexpected behavior that may be a bug.
-  - name: Feature suggestion or feedback
-    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
-    about: Start a discussion about your idea for improving Zulip.
-  - name: Issue with running or upgrading a Zulip server
-    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
-    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
-  - name: Other support requests and sales questions
-    url: https://zulip.com/help/contact-support
-    about: Contact us — we're happy to help!

.github/funding.json

@@ -1,82 +0,0 @@
-{
-    "version": "v1.0.0",
-    "entity": {
-        "type": "organisation",
-        "role": "steward",
-        "name": "Kandra Labs, Inc.",
-        "email": "support@zulip.com",
-        "description": "Guiding the Zulip community in developing a world-class organized team chat product with apps for every major desktop and mobile platform requires leadership from a talented, dedicated team. We believe that the only sustainable model is for our core team to be compensated fairly for their time. We have thus founded a company (Kandra Labs) to steward and financially support Zulip’s development. We are growing our business sustainably, without venture capital funding. VCs are incentivized to push companies to gamble for explosive growth. Often, the result is that a company with a useful product burns rapidly through its resources and goes out of business. We have built Zulip as a sustainable business (also supported by SBIR grants from the US National Science Foundation), and are being thoughtful about our pace of spending. Funding our company without venture capital also allows us to live by our values, without investor pressure to compromise them when doing so might be “good business” or “what everyone does”.",
-        "webpageUrl": {
-            "url": "https://zulip.com/values/",
-            "wellKnown": "https://zulip.com/.well-known/funding-manifest-urls"
-        }
-    },
-    "projects": [
-        {
-            "guid": "zulip",
-            "name": "Zulip",
-            "description": "Zulip is an open-source team chat application designed for seamless remote and hybrid work. With conversations organized by topic, Zulip is ideal for both live and asynchronous communication. Zulip’s 100% open-source software is available as a cloud service or a self-hosted solution, and is used by thousands of organizations around the world. An important part of Zulip’s mission is ensuring that worthy organizations, from programming-language developers to research communities, are able to use Zulip whether or not they have funding. For this reason, we sponsor Zulip Cloud Standard for open source projects, non-profits, education, and academic research. This program has grown exponentially since its inception; today we are proud to fully sponsor Zulip hosting for several hundred organizations. Support from the community will help us continue to afford these programs as their popularity grows.        ",
-            "webpageUrl": {
-                "url": "https://zulip.com/",
-                "wellKnown": "https://zulip.com/.well-known/funding-manifest-urls"
-            },
-            "repositoryUrl": {
-                "url": "https://github.com/zulip"
-            },
-            "licenses": ["spdx:Apache-2.0"],
-            "tags": ["communication", "team-chat", "collaboration"]
-        }
-    ],
-    "funding": {
-        "channels": [
-            {
-                "guid": "github-sponsors",
-                "type": "payment-provider",
-                "address": "https://github.com/sponsors/zulip",
-                "description": "Preferred channel for sponsoring Zulip, since GitHub Sponsors does not charge any fees to sponsored projects."
-            },
-            {
-                "guid": "patreon",
-                "type": "payment-provider",
-                "address": "https://patreon.com/zulip"
-            },
-            {
-                "guid": "open-collective",
-                "type": "payment-provider",
-                "address": "https://opencollective.com/zulip"
-            }
-        ],
-        "plans": [
-            {
-                "guid": "github-sponsors",
-                "status": "active",
-                "name": "Support Zulip",
-                "description": "Contribute to Zulip's development and free hosting for open source projects and other worthy organizations!",
-                "amount": 0,
-                "currency": "USD",
-                "frequency": "monthly",
-                "channels": ["github-sponsors"]
-            },
-            {
-                "guid": "patreon",
-                "status": "active",
-                "name": "Support Zulip",
-                "description": "Contribute to Zulip's development and free hosting for open source projects and other worthy organizations!",
-                "amount": 0,
-                "currency": "USD",
-                "frequency": "monthly",
-                "channels": ["patreon"]
-            },
-            {
-                "guid": "open-collective",
-                "status": "active",
-                "name": "Support Zulip",
-                "description": "Contribute to Zulip's development and free hosting for open source projects and other worthy organizations!",
-                "amount": 0,
-                "currency": "USD",
-                "frequency": "monthly",
-                "channels": ["open-collective"]
-            }
-        ]
-    }
-}

.github/pull_request_template.md

@@ -1,43 +1,11 @@
-<!-- Describe your pull request here.-->
+<!-- What's this PR for?  (Just a link to an issue is fine.) -->
 
-Fixes: <!-- Issue link, or clear description.-->
+**Testing plan:** <!-- How have you tested? -->
 
-<!-- If the PR makes UI changes, always include one or more still screenshots to demonstrate your changes. If it seems helpful, add a screen capture of the new functionality as well.
+**GIFs or screenshots:** <!-- If a UI change.  See:
+  https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
+  -->
 
-Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
--->
-
-**Screenshots and screen captures:**
-
-<details>
-<summary>Self-review checklist</summary>
-
-<!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
-https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
-
-<!-- Once you create the PR, check off all the steps below that you have completed.
-If any of these steps are not relevant or you have not completed, leave them unchecked.-->
-
-- [ ] [Self-reviewed](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code) the changes for clarity and maintainability
-      (variable names, code reuse, readability, etc.).
-
-Communicate decisions, questions, and potential concerns.
-
-- [ ] Explains differences from previous plans (e.g., issue description).
-- [ ] Highlights technical choices and bugs encountered.
-- [ ] Calls out remaining decisions and concerns.
-- [ ] Automated tests verify logic where appropriate.
-
-Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
-
-- [ ] Each commit is a coherent idea.
-- [ ] Commit message(s) explain reasoning and motivation for changes.
-
-Completed manual review and testing of the following:
-
-- [ ] Visual appearance of the changes.
-- [ ] Responsiveness and internationalization.
-- [ ] Strings and tooltips.
-- [ ] End-to-end functionality of buttons, interactions and flows.
-- [ ] Corner cases, error conditions, and easily imagined bugs.
-</details>
+<!-- Also be sure to make clear, coherent commits:
+  https://zulip.readthedocs.io/en/latest/contributing/version-control.html
+  -->

.github/workflows/cancel-previous-runs.yml

@@ -0,0 +1,43 @@
+name: Cancel previous runs
+on: [push, pull_request]
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  cancel:
+    name: Cancel previous runs
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+
+    # Don't run this job for zulip/zulip pushes since we
+    # want to run those jobs.
+    if: ${{ github.event_name != 'push' || github.event.repository.full_name != 'zulip/zulip' }}
+
+    steps:
+      # We get workflow IDs from GitHub API so we don't have to maintain
+      # a hard-coded list of IDs which need to be updated when a workflow
+      # is added or removed. And, workflow IDs are different for other forks
+      # so this is required.
+      - name: Get workflow IDs.
+        id: workflow_ids
+        continue-on-error: true # Don't fail this job on failure
+        env:
+          # This is in <owner>/<repo> format e.g. zulip/zulip
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          workflow_api_url=https://api.github.com/repos/$REPOSITORY/actions/workflows
+          curl $workflow_api_url -o workflows.json
+
+          script="const {workflows} = require('./workflows'); \
+                  const ids = workflows.map(workflow => workflow.id); \
+                  console.log(ids.join(','));"
+          ids=$(node -e "$script")
+          echo "::set-output name=ids::$ids"
+
+      - uses: styfle/cancel-workflow-action@0.9.0
+        continue-on-error: true # Don't fail this job on failure
+        with:
+          workflow_id: ${{ steps.workflow_ids.outputs.ids }}
+          access_token: ${{ github.token }}

.github/workflows/codeql-analysis.yml

@@ -1,40 +1,31 @@
 name: "Code scanning"
 
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-    branches: ["*.x", chat.zulip.org, main]
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-permissions:
-  contents: read
+on: [push, pull_request]
 
 jobs:
   CodeQL:
-    permissions:
-      actions: read # for github/codeql-action/init to get workflow details
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/analyze to upload SARIF results
     if: ${{!github.event.repository.private}}
     runs-on: ubuntu-latest
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v2
+        with:
+          # We must fetch at least the immediate parents so that if this is
+          # a pull request then we can check out the head.
+          fetch-depth: 2
+
+      # If this run was triggered by a pull request event, then check out
+      # the head of the pull request instead of the merge commit.
+      - run: git checkout HEAD^2
+        if: ${{ github.event_name == 'pull_request' }}
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v1
 
         # Override language selection by uncommenting this and choosing your languages
         # with:
         #   languages: go, javascript, csharp, python, cpp, java
-
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v1

.github/workflows/production-suite.yml

@@ -2,52 +2,43 @@ name: Zulip production suite
 
 on:
   push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
+    paths:
+      - "**/migrations/**"
+      - puppet/**
+      - requirements/**
+      - scripts/**
+      - static/**
+      - tools/**
+      - zproject/**
+      - yarn.lock
+      - .github/workflows/production-suite.yml
   pull_request:
     paths:
-      - .github/workflows/production-suite.yml
       - "**/migrations/**"
-      - manage.py
-      - pnpm-lock.yaml
       - puppet/**
+      - requirements/**
       - scripts/**
+      - static/**
       - tools/**
-      - uv.lock
-      - web/babel.config.js
-      - web/postcss.config.js
-      - web/third/**
-      - web/webpack.config.ts
-      - zerver/worker/queue_processors.py
-      - zerver/lib/push_notifications.py
-      - zerver/lib/storage.py
-      - zerver/decorator.py
       - zproject/**
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
+      - yarn.lock
+      - .github/workflows/production-suite.yml
 
 defaults:
   run:
     shell: bash
 
-permissions:
-  contents: read
-
 jobs:
   production_build:
     # This job builds a release tarball from the current commit, which
     # will be used for all of the following install/upgrade tests.
-    name: Ubuntu 22.04 production build
+    name: Bionic production build
     runs-on: ubuntu-latest
 
-    # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-    # the top explain how to build and upload these images.
-    # Ubuntu 22.04 ships with Python 3.10.12.
-    container: zulip/ci:jammy
-
+    # This docker image was created by a generated Dockerfile at:
+    #   tools/ci/images/bionic/Dockerfile
+    # Bionic ships with Python 3.6.
+    container: zulip/ci:bionic
     steps:
       - name: Add required permissions
         run: |
@@ -65,69 +56,56 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-emoji-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
-        uses: actions/cache@v4
+      - name: Restore node_modules cache
+        uses: actions/cache@v2
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-jammy-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ github.job }}-${{ hashFiles('package.json') }}-${{ hashFiles('yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ github.job }}
 
-      - name: Restore uv cache
-        uses: actions/cache@v4
+      - name: Restore python cache
+        uses: actions/cache@v2
         with:
-          path: ~/.cache/uv
-          key: uv-jammy-${{ hashFiles('uv.lock') }}
-          restore-keys: uv-jammy-
+          path: /srv/zulip-venv-cache
+          key: v1-venv-${{ github.job }}-${{ hashFiles('requirements/dev.txt') }}
+          restore-keys: v1-venv-${{ github.job }}
 
       - name: Restore emoji cache
-        uses: actions/cache@v4
+        uses: actions/cache@v2
         with:
           path: /srv/zulip-emoji-cache
-          key: v1-emoji-jammy-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
-          restore-keys: v1-emoji-jammy
+          key: v1-emoji-${{ github.job }}-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
+          restore-keys: v1-emoji-${{ github.job }}
+
+      - name: Do Bionic hack
+        run: |
+          # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
+          # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
+          sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
 
       - name: Build production tarball
         run: ./tools/ci/production-build
 
       - name: Upload production build artifacts for install jobs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v2
         with:
           name: production-tarball
           path: /tmp/production-build
-          retention-days: 1
+          retention-days: 14
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Minimize uv cache
-        run: uv cache prune --ci
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message
 
   production_install:
     # This job installs the server release tarball built above on a
@@ -137,22 +115,27 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 production install and PostgreSQL upgrade with pgroonga
-            os: jammy
-            extra-args: ""
+          # Base images are built using `tools/ci/Dockerfile.template`.
+          # The comments at the top explain how to build and upload these images.
+          - docker_image: zulip/ci:bionic
+            name: Bionic production install
+            is_bionic: true
+            os: bionic
 
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 production install
-            os: noble
-            extra-args: ""
+          - docker_image: zulip/ci:focal
+            name: Focal production install
+            is_focal: true
+            os: focal
 
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install with custom db name and user
-            os: bookworm
-            extra-args: --test-custom-db
+          - docker_image: zulip/ci:buster
+            name: Buster production install
+            is_buster: true
+            os: buster
+
+          - docker_image: zulip/ci:bullseye
+            name: Bullseye production install
+            is_bullseye: true
+            os: bullseye
 
     name: ${{ matrix.name  }}
     container:
@@ -163,7 +146,7 @@ jobs:
 
     steps:
       - name: Download built production tarball
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v2
         with:
           name: production-tarball
           path: /tmp
@@ -175,58 +158,61 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-          # Since actions/download-artifact@v4 loses all the permissions
+          # Create the zulip directory that the tools/ci/ scripts needs
+          mkdir -p /home/github/zulip
+
+          # Since actions/download-artifact@v2 loses all the permissions
           # of the tarball uploaded by the upload artifact fix those.
+          chmod +x /tmp/production-extract-tarball
           chmod +x /tmp/production-upgrade-pg
-          chmod +x /tmp/production-pgroonga
           chmod +x /tmp/production-install
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
+          chmod +x /tmp/send-failure-message
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-emoji-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
+      - name: Restore node_modules cache
+        uses: actions/cache@v2
+        with:
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('/tmp/package.json') }}-${{ hashFiles('/tmp/yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
+
+      - name: Do Bionic hack
+        if: ${{ matrix.is_bionic }}
+        run: |
+          # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
+          # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
+          sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
+
+      - name: Production extract tarball
+        run: /tmp/production-extract-tarball
+
       - name: Install production
-        run: sudo /tmp/production-install ${{ matrix.extra-args }}
+        run: |
+          sudo service rabbitmq-server restart
+          sudo /tmp/production-install
 
       - name: Verify install
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Install pgroonga
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-pgroonga
-
-      - name: Verify install after installing pgroonga
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
+        run: sudo /tmp/production-verify
 
       - name: Upgrade postgresql
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.is_bionic }}
         run: sudo /tmp/production-upgrade-pg
 
       - name: Verify install after upgrading postgresql
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
+        if: ${{ matrix.is_bionic }}
+        run: sudo /tmp/production-verify
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message
 
   production_upgrade:
     # The production upgrade job starts with a container with a
@@ -239,20 +225,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Docker images are built from 'tools/ci/Dockerfile.prod'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:jammy-6.0
-            name: 6.0 Version Upgrade
-            os: jammy
-          - docker_image: zulip/ci:bookworm-7.0
-            name: 7.0 Version Upgrade
-            os: bookworm
-          - docker_image: zulip/ci:bookworm-8.0
-            name: 8.0 Version Upgrade
-            os: bookworm
-          - docker_image: zulip/ci:noble-9.0
-            name: 9.0 Version Upgrade
-            os: noble
+          # Base images are built using `tools/ci/Dockerfile.prod.template`.
+          # The comments at the top explain how to build and upload these images.
+          - docker_image: zulip/ci:buster-3.4
+            name: 3.4 Version Upgrade
+            is_focal: true
+            os: buster
 
     name: ${{ matrix.name  }}
     container:
@@ -263,7 +241,7 @@ jobs:
 
     steps:
       - name: Download built production tarball
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v2
         with:
           name: production-tarball
           path: /tmp
@@ -275,17 +253,11 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-          # Since actions/download-artifact@v4 loses all the permissions
+          # Since actions/download-artifact@v2 loses all the permissions
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-emoji-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
+          chmod +x /tmp/send-failure-message
 
       - name: Upgrade production
         run: sudo /tmp/production-upgrade
@@ -296,19 +268,8 @@ jobs:
         # - name: Verify install
         #   run: sudo /tmp/production-verify
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message

.github/workflows/update-oneclick-apps.yml

@@ -2,14 +2,11 @@ name: Update one click apps
 on:
   release:
     types: [published]
-permissions:
-  contents: read
-
 jobs:
   update-digitalocean-oneclick-app:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
       - name: Update DigitalOcean one click app
         env:
           DIGITALOCEAN_API_KEY: ${{ secrets.ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY }}
@@ -22,6 +19,6 @@ jobs:
         run: |
           export PATH="$HOME/.local/bin:$PATH"
           git clone https://github.com/zulip/marketplace-partners
-          pip3 install python-digitalocean zulip fab-classic PyNaCl
+          pip3 install python-digitalocean zulip fab-classic
           echo $PATH
           python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

.github/workflows/zulip-ci.yml

@@ -4,49 +4,43 @@
 
 name: Zulip CI
 
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
+on: [push, pull_request]
 
 defaults:
   run:
     shell: bash
 
-permissions:
-  contents: read
-
 jobs:
   tests:
     strategy:
       fail-fast: false
       matrix:
         include:
-          # Base images are built using `tools/ci/Dockerfile.prod.template`.
-          # The comments at the top explain how to build and upload these images.
-          # Ubuntu 22.04 ships with Python 3.10.12.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 (Python 3.10, backend + frontend)
-            os: jammy
-            include_documentation_tests: false
+          # This docker image was created by a generated Dockerfile at:
+          #   tools/ci/images/bionic/Dockerfile
+          # Bionic ships with Python 3.6.
+          - docker_image: zulip/ci:bionic
+            name: Ubuntu 18.04 Bionic (Python 3.6, backend + frontend)
+            os: bionic
+            is_bionic: true
             include_frontend_tests: true
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend + documentation)
-            os: bookworm
-            include_documentation_tests: true
+
+          # This docker image was created by a generated Dockerfile at:
+          #   tools/ci/images/focal/Dockerfile
+          # Focal ships with Python 3.8.2.
+          - docker_image: zulip/ci:focal
+            name: Ubuntu 20.04 Focal (Python 3.8, backend)
+            os: focal
+            is_focal: true
             include_frontend_tests: false
-          # Ubuntu 24.04 ships with Python 3.12.2.
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 (Python 3.12, backend)
-            os: noble
-            include_documentation_tests: false
+
+          # This docker image was created by a generated Dockerfile at:
+          #   tools/ci/images/focal/Dockerfile
+          # Bullseye ships with Python 3.9.2.
+          - docker_image: zulip/ci:bullseye
+            name: Debian 11 Bullseye (Python 3.9, backend)
+            os: bullseye
+            is_bullseye: true
             include_frontend_tests: false
 
     runs-on: ubuntu-latest
@@ -54,7 +48,7 @@ jobs:
     container: ${{ matrix.docker_image }}
     env:
       # GitHub Actions sets HOME to /github/home which causes
-      # problem later in provision and frontend test that runs
+      # problem later in provison and frontend test that runs
       # tools/setup/postgresql-init-dev-db because of the .pgpass
       # location. PostgreSQL (psql) expects .pgpass to be at
       # /home/github/.pgpass and setting home to `/home/github/`
@@ -62,57 +56,115 @@ jobs:
       HOME: /home/github/
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Add required permissions
+        run: |
+          # The checkout actions doesn't clone to ~/zulip or allow
+          # us to use the path option to clone outside the current
+          # /__w/zulip/zulip directory. Since this directory is owned
+          # by root we need to change it's ownership to allow the
+          # github user to clone the code here.
+          # Note: /__w/ is a docker volume mounted to $GITHUB_WORKSPACE
+          # which is /home/runner/work/.
+          sudo chown -R github .
+
+          # This is the GitHub Actions specific cache directory the
+          # the current github user must be able to access for the
+          # cache action to work. It is owned by root currently.
+          sudo chmod -R 0777 /__w/_temp/
+
+      - uses: actions/checkout@v2
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-emoji-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
-        uses: actions/cache@v4
+      - name: Restore node_modules cache
+        uses: actions/cache@v2
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('package.json') }}-${{ hashFiles('yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
 
-      - name: Restore uv cache
-        uses: actions/cache@v4
+      - name: Restore python cache
+        uses: actions/cache@v2
         with:
-          path: ~/.cache/uv
-          key: uv-${{ matrix.os }}-${{ hashFiles('uv.lock') }}
-          restore-keys: uv-${{ matrix.os }}-
+          path: /srv/zulip-venv-cache
+          key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/dev.txt') }}
+          restore-keys: v1-venv-${{ matrix.os }}
 
       - name: Restore emoji cache
-        uses: actions/cache@v4
+        uses: actions/cache@v2
         with:
           path: /srv/zulip-emoji-cache
-          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json', 'tools/setup/emoji/build_emoji', 'tools/setup/emoji/emoji_setup_utils.py', 'tools/setup/emoji/emoji_names.py', 'package.json') }}
+          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
           restore-keys: v1-emoji-${{ matrix.os }}
 
+      - name: Do Bionic hack
+        if: ${{ matrix.is_bionic }}
+        run: |
+          # Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
+          # https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
+          sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
+
       - name: Install dependencies
         run: |
           # This is the main setup job for the test suite
           ./tools/ci/setup-backend --skip-dev-db-build
-          scripts/lib/clean_unused_caches.py --verbose --threshold=0
+
+          # Cleaning caches is mostly unnecessary in GitHub Actions, because
+          # most builds don't get to write to the cache.
+          # scripts/lib/clean-unused-caches --verbose --threshold 0
 
       - name: Run tools test
         run: |
           source tools/ci/activate-venv
           ./tools/test-tools
 
-      - name: Run Codespell lint
+      - name: Run backend lint
         run: |
           source tools/ci/activate-venv
-          ./tools/run-codespell
+          echo "Test suite is running under $(python --version)."
+          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
+
+      - name: Run frontend lint
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
+
+      - name: Run backend tests
+        run: |
+          source tools/ci/activate-venv
+          ./tools/test-backend --coverage --include-webhooks --no-cov-cleanup --ban-console-output
+
+      - name: Run mypy
+        run: |
+          source tools/ci/activate-venv
+          # We run mypy after the backend tests so we get output from the
+          # backend tests, which tend to uncover more serious problems, first.
+          ./tools/run-mypy --version
+          ./tools/run-mypy
+
+      - name: Run miscellaneous tests
+        run: |
+          source tools/ci/activate-venv
+
+          # Currently our compiled requirements files will differ for different python versions
+          # so we will run test-locked-requirements only for Bionic.
+          # ./tools/test-locked-requirements
+          # ./tools/test-run-dev  # https://github.com/zulip/zulip/pull/14233
+          #
+          # This test has been persistently flaky at like 1% frequency, is slow,
+          # and is for a very specific single feature, so we don't run it by default:
+          # ./tools/test-queue-worker-reload
+
+          ./tools/test-migrations
+          ./tools/setup/optimize-svg --check
+          ./tools/setup/generate_integration_bots_avatars.py --check-missing
 
-      # We run the tests that are only run in a specific job early, so
-      # that we get feedback to the developer about likely failures as
-      # quickly as possible. Backend/mypy failures that aren't
-      # identical across different versions are much more rare than
-      # frontend linter or node test failures.
       - name: Run documentation and api tests
-        if: ${{ matrix.include_documentation_tests }}
         run: |
           source tools/ci/activate-venv
           # In CI, we only test links we control in test-documentation to avoid flakes
@@ -125,13 +177,7 @@ jobs:
         run: |
           source tools/ci/activate-venv
           # Run the node tests first, since they're fast and deterministic
-          ./tools/test-js-with-node --coverage --parallel=1
-
-      - name: Run frontend lint
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
+          ./tools/test-js-with-node --coverage
 
       - name: Check schemas
         if: ${{ matrix.include_frontend_tests }}
@@ -154,52 +200,6 @@ jobs:
           source tools/ci/activate-venv
           ./tools/test-js-with-puppeteer
 
-      - name: Check pnpm dedupe
-        if: ${{ matrix.include_frontend_tests }}
-        run: pnpm dedupe --check
-
-      - name: Run backend lint
-        run: |
-          source tools/ci/activate-venv
-          echo "Test suite is running under $(python --version)."
-          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
-
-      - name: Run backend tests
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-backend ${{ matrix.os != 'bookworm' && '--coverage' || '' }} --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
-
-      - name: Run mypy
-        run: |
-          source tools/ci/activate-venv
-          # We run mypy after the backend tests so we get output from the
-          # backend tests, which tend to uncover more serious problems, first.
-          ./tools/run-mypy --version
-          ./tools/run-mypy
-
-      - name: Run miscellaneous tests
-        run: |
-          source tools/ci/activate-venv
-          uv lock --check
-
-          # ./tools/test-run-dev  # https://github.com/zulip/zulip/pull/14233
-          #
-          # This test has been persistently flaky at like 1% frequency, is slow,
-          # and is for a very specific single feature, so we don't run it by default:
-          # ./tools/test-queue-worker-reload
-
-          ./tools/test-migrations
-          ./tools/setup/optimize-svg --check
-          ./tools/setup/generate_integration_bots_avatars.py --check-missing
-          ./tools/ci/check-executables
-
-          # Ban check-database-compatibility from transitively
-          # relying on static/generated, because it might not be
-          # up-to-date at that point in upgrade-zulip-stage-2.
-          chmod 000 static/generated web/generated
-          ./scripts/lib/check-database-compatibility
-          chmod 755 static/generated web/generated
-
       - name: Check for untracked files
         run: |
           source tools/ci/activate-venv
@@ -211,50 +211,40 @@ jobs:
               exit 1
           fi
 
+      - name: Test locked requirements
+        if: ${{ matrix.is_bionic }}
+        run: |
+          . /srv/zulip-py3-venv/bin/activate && \
+          ./tools/test-locked-requirements
+
       - name: Upload coverage reports
 
         # Only upload coverage when both frontend and backend
-        # tests are run.
+        # tests are ran.
         if: ${{ matrix.include_frontend_tests }}
-        uses: codecov/codecov-action@v4
-        with:
-          files: var/coverage.xml,var/node-coverage/lcov.info
-          token: ${{ secrets.CODECOV_TOKEN }}
+        run: |
+          # Codcov requires `.coverage` file to be stored in the
+          # current working directory.
+          mv ./var/.coverage ./.coverage
+          . /srv/zulip-py3-venv/bin/activate || true
+
+          pip install codecov && codecov || echo "Error in uploading coverage reports to codecov.io."
 
       - name: Store Puppeteer artifacts
         # Upload these on failure, as well
         if: ${{ always() && matrix.include_frontend_tests }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v2
         with:
           name: puppeteer
           path: ./var/puppeteer
           retention-days: 60
 
       - name: Check development database build
+        if: ${{ matrix.is_focal || matrix.is_bullseye }}
         run: ./tools/ci/setup-backend
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Minimize uv cache
-        run: uv cache prune --ci
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message

.gitignore

@@ -17,25 +17,25 @@
 # See `git help ignore` for details on the format.
 
 ## Config files for the dev environment
-/zproject/apns-dev.pem
-/zproject/apns-dev-key.p8
 /zproject/dev-secrets.conf
-/zproject/custom_dev_settings.py
 /tools/conf.ini
 /tools/custom_provision
 /tools/droplets/conf.ini
 
 ## Byproducts of setting up and using the dev environment
 *.pyc
-*.tsbuildinfo
 package-lock.json
 
 /.vagrant
-/var
+/var/*
+!/var/puppeteer
+/var/puppeteer/*
+!/var/puppeteer/test_credentials.d.ts
 
 /.dmypy.json
-/.ruff_cache
-/.venv
+
+# Dockerfiles generated for continuous integration
+/tools/ci/images
 
 # Generated i18n data
 /locale/en
@@ -46,11 +46,11 @@ package-lock.json
 # Static build
 *.mo
 npm-debug.log
-/.pnpm-store
 /node_modules
 /prod-static
 /staticfiles.json
 /webpack-stats-production.json
+/yarn-error.log
 zulip-git-version
 
 # Test / analysis tools
@@ -58,6 +58,8 @@ zulip-git-version
 
 ## Files (or really symlinks) created in a prod deployment
 /zproject/prod_settings.py
+/zulip-current-venv
+/zulip-py3-venv
 
 ## Files left by various editors and local environments
 # (Ideally these should be in everyone's respective personal gitignore files.)
@@ -71,21 +73,15 @@ zulip.kdev4
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
+.vscode/
 *.DS_Store
-# VS Code. Avoid checking in .vscode in general, while still specifying
-# recommended extensions for working with this repository.
-/.vscode/**/*
-!/.vscode/extensions.json
-# .cache/ is generated by VS Code test runner
+# .cache/ is generated by Visual Studio Code's test runner
 .cache/
 .eslintcache
 
 # Core dump files
 core
 
-# Static generated files for landing page.
-/static/images/landing-page/hello/generated
-
 ## Miscellaneous
 # (Ideally this section is empty.)
 .transifexrc

.gitlint

@@ -1,13 +1,13 @@
 [general]
 ignore=title-trailing-punctuation, body-min-length, body-is-missing
 
-extra-path=tools/lib/gitlint_rules.py
+extra-path=tools/lib/gitlint-rules.py
 
 [title-match-regex]
 regex=^(.+:\ )?[A-Z].+\.$
 
 [title-max-length]
-line-length=72
+line-length=76
 
 [body-max-line-length]
 line-length=76

.mailmap

@@ -1,164 +1,40 @@
-# This file teaches `git log` and friends the canonical names
-# and email addresses to use for our contributors.
-#
-# For details on the format, see:
-#   https://git.github.io/htmldocs/gitmailmap.html
-#
-# Handy commands for examining or adding to this file:
-#
-#     # shows all names/emails after mapping, sorted:
-#   $ git shortlog -es | sort -k2
-#
-#     # shows raw names/emails, filtered by mapped name:
-#   $ git log --format='%an %ae' --author=$NAME | uniq -c
-
-acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
-acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
-Adam Benesh <Adam.Benesh@gmail.com>
-Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
-Adarsh Tiwari <xoldyckk@gmail.com>
-Aditya Chaudhary <aditya.chaudhary1558@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com> <78212328+adnan-td@users.noreply.github.com>
 Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
 Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
-Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
-Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
-Aman Agrawal <amanagr@zulip.com>
 Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
 Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
 Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Aryan Bhokare <aryan1bhokare@gmail.com>
-Aryan Bhokare <aryan1bhokare@gmail.com> <92683836+aryan-bhokare@users.noreply.github.com>
-Aryan Shridhar <aryanshridhar7@gmail.com>
-Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
-Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
-Austin Riba <austin@zulip.com> <austin@m51.io>
-Bedo Khaled <bedokhaled66@gmail.com>
-Bedo Khaled <bedokhaled66@gmail.com> <64221784+abdelrahman725@users.noreply.github.com>
-BIKI DAS <bikid475@gmail.com>
-Brijmohan Siyag <brijsiyag@gmail.com>
-Brock Whittaker <whittakerbrock@gmail.com> <bjwhitta@asu.edu>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.com>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulipchat.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brockwhittaker@Brocks-MacBook.local>
+Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
+Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
+Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
 Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-codewithnick <nikhilsingh526452@gmail.com>
-Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dhruv Goyal <dhruvgoyal.dev@gmail.com>
-Dinesh <chdinesh1089@gmail.com>
-Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
-Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
-Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
-Evy Kassirer <evy@zulip.com>
-Evy Kassirer <evy@zulip.com> <evy.kassirer@gmail.com>
-Evy Kassirer <evy@zulip.com> <evykassirer@users.noreply.github.com>
-Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
 Greg Price <greg@zulip.com> <gnprice@gmail.com>
 Greg Price <greg@zulip.com> <greg@zulipchat.com>
 Greg Price <greg@zulip.com> <price@mit.edu>
-Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
-Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
-Jai soni <jai_s@me.iitr.ac.in>
-Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-John Lu <JohnLu10212004@gmail.com>
-John Lu <JohnLu10212004@gmail.com> <87673068+JohnLu2004@users.noreply.github.com>
-Joseph Ho <josephho678@gmail.com>
-Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
-Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
-Karl Stolley <karl@zulip.com> <karl@stolley.dev>
 Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
 Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
-Kevin Scott <kevin.scott.98@gmail.com>
-Kislay Verma <kislayuv27@gmail.com>
-Kunal Sharma <v.shm.kunal@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com> <lalits01@smartek21.com>
-Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
-Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
-Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
-Matt Keller <matt@zulip.com>
-Matt Keller <matt@zulip.com> <m@cognusion.com>
-Nehal Sharma <bablinaneh@gmail.com>
-Nehal Sharma <bablinaneh@gmail.com> <68962290+N-Shar-ma@users.noreply.github.com>
-Nimish Medatwal <medatwalnimish@gmail.com>
-Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
-nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
-Palash Baderia <palash.baderia@outlook.com>
-Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
-Palash Raghuwanshi <singhpalash0@gmail.com>
-Parth <mittalparth22@gmail.com>
-Pratik Chanda <pratikchanda2000@gmail.com>
-Pratik Solanki <pratiksolanki2021@gmail.com>
-Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
 Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
-Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
-Rein Zustand (rht) <rhtbot@protonmail.com>
-Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
 Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
 Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
-Rixant Rokaha <rixantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Rohan Gudimetla <rohan.gudimetla07@gmail.com>
-Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
-Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Sanchit Sharma <ssharmas10662@gmail.com>
-Satyam Bansal <sbansal1999@gmail.com>
-Sayam Samal <samal.sayam@gmail.com>
+Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
-Shashank Singh <21bec103@iiitdmj.ac.in>
-Shlok Patel <shlokcpatel2001@gmail.com>
-Shu Chen <shu@zulip.com>
-Shubham Padia <shubham@zulip.com>
-Shubham Padia <shubham@zulip.com> <shubham@glints.com>
-Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
 Steve Howell <showell@zulip.com> <showell30@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@zulipchat.com>
 Steve Howell <showell@zulip.com> <steve@humbughq.com>
 Steve Howell <showell@zulip.com> <steve@zulip.com>
-strifel <info@strifel.de>
-Sujal Shah <sujalshah28092004@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com> <133781250+tnmkr@users.noreply.github.com>
-Tim Abbott <tabbott@zulip.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
 Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Tomasz Kolek <tomasz-kolek@o2.pl> <tomasz-kolek@go2.pl>
-Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
-umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
-umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
-Viktor Illmer <1476338+v-ji@users.noreply.github.com>
 Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
 Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
-Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
-Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
-Zev Benjamin <zev@zulip.com> <zev@mit.edu>
-Zixuan James Li <p359101898@gmail.com>
-Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
-Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>
+Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>

.prettierignore

@@ -1,17 +1,8 @@
-pnpm-lock.yaml
-/api_docs/**/*.md
 /corporate/tests/stripe_fixtures
-/help/**/*.md
 /locale
+/static/third
 /templates/**/*.md
 /tools/setup/emoji/emoji_map.json
-/web/third/*
-!/web/third/marked
-/web/third/marked/*
-!/web/third/marked/lib
-/web/third/marked/lib/*
-!/web/third/marked/lib/marked.d.cts
 /zerver/tests/fixtures
 /zerver/webhooks/*/doc.md
-/zerver/webhooks/github/githubsponsors.md
 /zerver/webhooks/*/fixtures

.pyre_configuration

@@ -0,0 +1,15 @@
+{
+  "source_directories": ["."],
+  "taint_models_path": [
+      "stubs/taint",
+      "zulip-py3-venv/lib/pyre_check/taint/"
+  ],
+  "search_path": [
+      "stubs/",
+      "zulip-py3-venv/lib/pyre_check/stubs/"
+  ],
+  "typeshed": "zulip-py3-venv/lib/pyre_check/typeshed/",
+  "exclude": [
+      "/srv/zulip/zulip-py3-venv/.*"
+  ]
+}

.readthedocs.yaml

@@ -1,19 +0,0 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-  jobs:
-    create_environment:
-      - asdf plugin add uv
-      - asdf install uv 0.6.2
-      - asdf global uv 0.6.2
-      - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv venv
-    install:
-      - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --only-group=docs
-
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true

.tx/config

@@ -1,39 +1,32 @@
-# Migrated from transifex-client format with `tx migrate`
-#
-# See https://developers.transifex.com/docs/using-the-client which hints at
-# this format, but in general, the headings are in the format of:
-#
-# [o:<org>:p:<project>:r:<resource>]
-
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans
+lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
-[o:zulip:p:zulip:r:djangopo]
+[zulip.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip:r:mobile]
+[zulip.translationsjson]
+file_filter = locale/<lang>/translations.json
+source_file = locale/en/translations.json
+source_lang = en
+type = KEYVALUEJSON
+
+[zulip.mobile]
 file_filter = locale/<lang>/mobile.json
 source_file = locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 
-[o:zulip:p:zulip:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip-test:r:djangopo]
+[zulip-test.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip-test:r:translationsjson]
+[zulip-test.translationsjson]
 file_filter = locale/<lang>/translations.json
 source_file = locale/en/translations.json
 source_lang = en

.vscode/extensions.json

@@ -1,23 +0,0 @@
-{
-    // Recommended VS Code extensions for zulip/zulip.
-    //
-    // VS Code prompts a user to install the recommended extensions
-    // when a workspace is opened for the first time.  The user can
-    // also review the list with the 'Extensions: Show Recommended
-    // Extensions' command.  See
-    // https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions
-    // for more information.
-    //
-    // Extension identifier format: ${publisher}.${name}.
-    // Example: vscode.csharp
-
-    "recommendations": [
-        "42crunch.vscode-openapi",
-        "dbaeumer.vscode-eslint",
-        "esbenp.prettier-vscode",
-        "ms-vscode-remote.vscode-remote-extensionpack"
-    ],
-
-    // Extensions recommended by VS Code which are not recommended for users of zulip/zulip.
-    "unwantedRecommendations": []
-}

.yarnrc

@@ -0,0 +1 @@
+ignore-scripts true

CODE_OF_CONDUCT.md

@@ -66,7 +66,7 @@ organizers may take any action they deem appropriate, up to and including a
 temporary ban or permanent expulsion from the community without warning (and
 without refund in the case of a paid event).
 
-If someone outside the development community (e.g., a user of the Zulip
+If someone outside the development community (e.g. a user of the Zulip
 software) engages in unacceptable behavior that affects someone in the
 community, we still want to know. Even if we don't have direct control over
 the violator, the community organizers can still support the people
@@ -102,72 +102,3 @@ This Code of Conduct is adapted from the
 under a
 [Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
 license.
-
-## Moderating the Zulip community
-
-Anyone can help moderate the Zulip community by helping make sure that folks are
-aware of the [community guidelines](https://zulip.com/development-community/)
-and this Code of Conduct, and that we maintain a positive and respectful
-atmosphere.
-
-Here are some guidelines for you how can help:
-
-- Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
-  and just trying to keep the atmosphere warm make the whole community function
-  more smoothly. New participants who feel accepted, listened to and respected
-  are likely to treat others the same way.
-
-- Be familiar with the [community
-  guidelines](https://zulip.com/development-community/), and cite them liberally
-  when a user violates them. Be polite but firm. Some examples:
-
-  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
-    you ask a question. As noted in the [guidelines for this
-    community](https://zulip.com/development-community/):
-
-    > Use @-mentions sparingly… there is generally no need to @-mention a
-    > core contributor unless you need their timely attention.
-
-  - @user, please keep in mind the following [community
-    guideline](https://zulip.com/development-community/):
-
-    > Don’t ask the same question in multiple places. Moderators read every
-    > public stream, and make sure every question gets a reply.
-
-    I’ve gone ahead and moved the other copy of this message to this thread.
-
-  - If asked a question in a direct message that is better discussed in a public
-    stream:
-    > Hi @user! Please start by reviewing
-    > https://zulip.com/development-community/#community-norms to learn how to
-    > get help in this community.
-
-- Users sometimes think chat.zulip.org is a testing instance. When this happens,
-  kindly direct them to use the **#test here** stream.
-
-- If you see a message that’s posted in the wrong place, go ahead and move it if
-  you have permissions to do so, even if you don’t plan to respond to it.
-  Leaving the “Send automated notice to new topic” option enabled helps make it
-  clear what happened to the person who sent the message.
-
-  If you are responding to a message that's been moved, mention the user in your
-  reply, so that the mention serves as a notification of the new location for
-  their conversation.
-
-- If a user is posting spam, please report it to an administrator. They will:
-
-  - Change the user's name to `<name> (spammer)` and deactivate them.
-  - Delete any spam messages they posted in public streams.
-
-- We care very much about maintaining a respectful tone in our community. If you
-  see someone being mean or rude, point out that their tone is inappropriate,
-  and ask them to communicate their perspective in a respectful way in the
-  future. If you don’t feel comfortable doing so yourself, feel free to ask a
-  member of Zulip's core team to take care of the situation.
-
-- Try to assume the best intentions from others (given the range of
-  possibilities presented by their visible behavior), and stick with a friendly
-  and positive tone even when someone’s behavior is poor or disrespectful.
-  Everyone has bad days and stressful situations that can result in them
-  behaving not their best, and while we should be firm about our community
-  rules, we should also enforce them with kindness.

CONTRIBUTING.md

@@ -1,418 +1,347 @@
-# Contributing guide
+# Contributing to Zulip
 
-Welcome! This is a step-by-step guide on how to get started contributing code to
-the [Zulip](https://zulip.com/) organized team chat [open-source
-project](https://github.com/zulip). Thousands of people use Zulip every day, and
-your work on Zulip will have a meaningful impact on their experience. We hope
-you'll join us!
+Welcome to the Zulip community!
 
-To learn about ways to contribute without writing code, please see our
-suggestions for how you can [support the Zulip
-project](https://zulip.com/help/support-zulip-project).
+## Community
 
-## Learning from the docs
+The
+[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
+is the primary communication forum for the Zulip community. It is a good
+place to start whether you have a question, are a new contributor, are a new
+user, or anything else. Make sure to read the
+[community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
+before posting. The Zulip community is also governed by a
+[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
 
-Zulip has a documentation-based approach to onboarding new contributors. As you
-are getting started, this page will be your go-to for figuring out what to do
-next. You will also explore other guides, learning about how to put together
-your first pull request, diving into [Zulip's
-subsystems](https://zulip.readthedocs.io/en/latest/subsystems/index.html), and
-much more. We hope you'll find this process to be a great learning experience.
+You can subscribe to
+[zulip-devel-announce@googlegroups.com](https://groups.google.com/g/zulip-devel-announce)
+or our [Twitter](https://twitter.com/zulip) account for a very low
+traffic (<1 email/month) way to hear about things like mentorship
+opportunities with Google Summer of Code, in-person sprints at
+conferences, and other opportunities to contribute.
 
-This page will guide you through the following steps:
+## Ways to contribute
 
-1. [Getting started](#getting-started)
-1. [Finding an issue to work on](#finding-an-issue-to-work-on)
-1. [Getting help](#getting-help) as you work on your first pull request
-1. Learning [what makes a great Zulip contributor](#what-makes-a-great-zulip-contributor)
-1. [Submitting a pull request](#submitting-a-pull-request)
-1. [Going beyond the first issue](#beyond-the-first-issue)
+To make a code or documentation contribution, read our
+[step-by-step guide](#your-first-codebase-contribution) to getting
+started with the Zulip codebase. A small sample of the type of work that
+needs doing:
 
-Any time you feel lost, come back to this guide. The information you need is
-likely somewhere on this page (perhaps in the list of [common
-questions](#common-questions)), or in one of the many references it points to.
+- Bug squashing and feature development on our Python/Django
+  [backend](https://github.com/zulip/zulip), web
+  [frontend](https://github.com/zulip/zulip), React Native
+  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
+  [desktop app](https://github.com/zulip/zulip-desktop).
+- Building out our
+  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
+- [Writing an integration](https://zulip.com/api/integrations-overview).
+- Improving our [user](https://zulip.com/help/) or
+  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
+- [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
+  and manually testing pull requests.
 
-If you've done all you can with the documentation and are still feeling stuck,
-join the [Zulip development community](https://zulip.com/development-community/)
-to ask for help! Before you post, be sure to review [community
-norms](https://zulip.com/development-community/#community-norms) and [where to
-post](https://zulip.com/development-community/#where-do-i-send-my-message) your
-question. The Zulip community is governed by a [code of
-conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
+**Non-code contributions**: Some of the most valuable ways to contribute
+don't require touching the codebase at all. We list a few of them below:
 
-## Getting started
+- [Reporting issues](#reporting-issues), including both feature requests and
+  bug reports.
+- [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
+- [Sponsor Zulip](https://github.com/sponsors/zulip) through the GitHub sponsors program.
+- [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
+  Zulip.
+- [Outreach](#zulip-outreach): Star us on GitHub, upvote us
+  on product comparison sites, or write for [the Zulip blog](https://blog.zulip.org/).
 
-### Learning how to use Git (the Zulip way)
+## Your first (codebase) contribution
 
-Zulip uses GitHub for source control and code review, and becoming familiar with
-Git is essential for navigating and contributing to the Zulip codebase. [Our
-guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html) will help
-you get started even if you've never used Git before.
+This section has a step by step guide to starting as a Zulip codebase
+contributor. It's long, but don't worry about doing all the steps perfectly;
+no one gets it right the first time, and there are a lot of people available
+to help.
 
-If you're familiar with Git, you'll still want to take a look at [our
-Zulip-specific Git
-tools](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
+- First, make an account on the
+  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
+  paying special attention to the community norms. If you'd like, introduce
+  yourself in
+  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
+  your name as the topic. Bonus: tell us about your first impressions of
+  Zulip, and anything that felt confusing/broken as you started using the
+  product.
+- Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
+- [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
+  getting help in
+  [#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
+  if you run into any troubles.
+- Read the
+  [Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
+  and do the Git tutorial (coming soon) if you are unfamiliar with
+  Git, getting help in
+  [#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
+  you run into any troubles. Be sure to check out the
+  [extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
 
-### Setting up your development environment and diving in
+### Picking an issue
 
-To get started contributing code to Zulip, you will need to set up the
-development environment for the Zulip codebase you want to work on. You'll then
-want to take some time to familiarize yourself with the code.
+Now, you're ready to pick your first issue! There are hundreds of open issues
+in the main codebase alone. This section will help you find an issue to work
+on.
 
-#### Server and web app
-
-1. [Install the development
-   environment](https://zulip.readthedocs.io/en/latest/development/overview.html).
-1. Familiarize yourself with [using the development
-   environment](https://zulip.readthedocs.io/en/latest/development/using.html).
-1. Go through the [new application feature
-   tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html)
-   to get familiar with how the Zulip codebase is organized and how to find code
-   in it.
-
-#### Flutter-based mobile app
-
-1. Set up a development environment following the instructions in [the project
-   README](https://github.com/zulip/zulip-flutter).
-1. Start reading recent commits to see the code we're writing.
-   Use either a [graphical Git viewer][] like `gitk`, or `git log -p`
-   with [the "secret" to reading its output][git-log-secret].
-1. Pick some of the code that appears in those Git commits and that looks
-   interesting. Use your IDE to visit that code and to navigate to related code,
-   reading to see how it works and how the codebase is organized.
-
-[graphical Git viewer]: https://zulip.readthedocs.io/en/latest/git/setup.html#get-a-graphical-client
-[git-log-secret]: https://github.com/zulip/zulip-mobile/blob/main/docs/howto/git.md#git-log-secret
-
-#### Desktop app
-
-Follow [this
-documentation](https://github.com/zulip/zulip-desktop/blob/main/development.md)
-to set up the Zulip Desktop development environment.
-
-#### Terminal app
-
-Follow [this
-documentation](https://github.com/zulip/zulip-terminal?tab=readme-ov-file#setting-up-a-development-environment)
-to set up the Zulip Terminal development environment.
-
-## Finding an issue to work on
-
-### Where to look for an issue
-
-Now you're ready to pick your first issue! Zulip has several repositories you
-can check out, depending on your interests. There are hundreds of open issues in
-the [main Zulip server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-alone.
-
-You can look through issues tagged with the "help wanted" label, which is used
-to indicate the issues that are open for contributions. You'll be able to claim
-unassigned issues, which you can find using the `no:assignee` filter in GitHub.
-You can also pick up issues that are assigned but are no longer being worked on.
-
-Some repositories use the "good first issue" label to tag issues that are
-especially approachable for new contributors.
-
-Here are some handy links for issues to look through:
-
-- [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- Mobile apps: no "help wanted" label, but see the
-  [project board](https://github.com/orgs/zulip/projects/5/views/4)
-  for the upcoming Flutter-based app. Look for issues up through the
-  "Launch" milestone, and that aren't already assigned.
-- [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
-- [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-
-### Picking an issue to work on
-
-There's a lot to learn while making your first pull request, so start small!
-Many first contributions have fewer than 10 lines of changes (not counting
-changes to tests).
-
-We recommend the following process for finding an issue to work on:
-
-1. Find an issue tagged with the "help wanted" label that is either unassigned,
-   or looks to be abandoned.
-1. Read the description of the issue and make sure you understand it.
-1. If it seems promising, poke around the product
-   (on [chat.zulip.org](https://chat.zulip.org) or in the development
-   environment) until you know how the piece being
-   described fits into the bigger picture. If after some exploration the
-   description seems confusing or ambiguous, post a question on the GitHub
-   issue, as others may benefit from the clarification as well.
-1. When you find an issue you like, try to get started working on it. See if you
-   can find the part of the code you'll need to modify (`git grep` is your
-   friend!) and get some idea of how you'll approach the problem.
-1. If you feel lost, that's OK! Go through these steps again with another issue.
-   There's plenty to work on, and the exploration you do will help you learn
-   more about the project.
-
-An assigned issue can be considered abandoned if:
-
-- There is no recent contributor activity.
-- There are no open PRs, or an open PR needs work in order to be ready for
-  review. For example, a PR may need to be updated to address reviewer feedback
-  or to pass tests.
-
-Note that you are _not_ claiming an issue while you are iterating through steps
-1-4. _Before you claim an issue_, you should be confident that you will be able to
-tackle it effectively.
-
-Additional tips for the [main server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
-
-- We especially recommend browsing recently opened issues, as there are more
-  likely to be easy ones for you to find.
-- Take a look at issues with the ["good first issue"
-  label](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22),
-  as they are especially accessible to new contributors. However, you will
-  likely find issues without this label that are accessible as well.
-- All issues are partitioned into areas like
+- If you're interested in
+  [mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
+  [desktop](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue),
+  or
+  [bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
+  development, check the respective links for open issues, or post in
+  [#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
+  [#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
+  [#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
+- For the main server and web repository, we recommend browsing
+  recently opened issues to look for issues you are confident you can
+  fix correctly in a way that clearly communicates why your changes
+  are the correct fix. Our GitHub workflow bot, zulipbot, limits
+  users who have 0 commits merged to claiming a single issue labeled
+  with "good first issue" or "help wanted".
+- We also partition all of our issues in the main repo into areas like
   admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
   through our [list of labels](https://github.com/zulip/zulip/labels), and
   click on some of the `area:` labels to see all the issues related to your
   areas of interest.
-- Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are highly confident you can resolve the
-  issue correctly and completely.
+- If the lists of issues are overwhelming, post in
+  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
+  bit about your background and interests, and we'll help you out. The most
+  important thing to say is whether you're looking for a backend (Python),
+  frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
+  documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
+  bit about your programming experience and available time.
 
-### Claiming an issue
+We also welcome suggestions of features that you feel would be valuable or
+changes that you feel would make Zulip a better open source project. If you
+have a new feature you'd like to add, we recommend you start by posting in
+[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
+feature idea and the problem that you're hoping to solve.
 
-#### In the main server/web app repository and Zulip Terminal repository
+Other notes:
 
-The Zulip server/web app repository
-([`zulip/zulip`](https://github.com/zulip/zulip/)) and the Zulip Terminal
-repository ([`zulip/zulip-terminal`](https://github.com/zulip/zulip-terminal/))
-are set up with a GitHub workflow bot called
-[Zulipbot](https://github.com/zulip/zulipbot), which manages issues and pull
-requests in order to create a better workflow for Zulip contributors.
+- For a first pull request, it's better to aim for a smaller contribution
+  than a bigger one. Many first contributions have fewer than 10 lines of
+  changes (not counting changes to tests).
+- The full list of issues explicitly looking for a contributor can be
+  found with the
+  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
+  and
+  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+  labels. Avoid issues with the "difficult" label unless you
+  understand why it is difficult and are confident you can resolve the
+  issue correctly and completely. Issues without one of these labels
+  are fair game if Tim has written a clear technical design proposal
+  in the issue, or it is a bug that you can reproduce and you are
+  confident you can fix the issue correctly.
+- For most new contributors, there's a lot to learn while making your first
+  pull request. It's OK if it takes you a while; that's normal! You'll be
+  able to work a lot faster as you build experience.
 
-To claim an issue in these repositories, simply post a comment that says
-`@zulipbot claim` to the issue thread. If the issue is [tagged with a help
-wanted label and is not assigned to someone
-else](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22+no%3Aassignee),
-Zulipbot will immediately assign the issue to you.
+### Working on an issue
 
-Note that new contributors can only claim one issue until their first pull request is
-merged. This is to encourage folks to finish ongoing work before starting
-something new. If you would like to pick up a new issue while waiting for review
-on an almost-ready pull request, you can post a comment to this effect on the
-issue you're interested in.
+To work on an issue, claim it by adding a comment with `@zulipbot claim` to
+the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
+workflow bot; it will assign you to the issue and label the issue as "in
+progress". Some additional notes:
 
-#### In other Zulip repositories
+- You can only claim issues with the
+  [good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
+  or
+  [help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
+  labels. Zulipbot will give you an error if you try to claim an issue
+  without one of those labels.
+- You're encouraged to ask questions on how to best implement or debug your
+  changes -- the Zulip maintainers are excited to answer questions to help
+  you stay unblocked and working efficiently. You can ask questions on
+  chat.zulip.org, or on the GitHub issue or pull request.
+- We encourage early pull requests for work in progress. Prefix the title of
+  work in progress pull requests with `[WIP]`, and remove the prefix when
+  you think it might be mergeable and want it to be reviewed.
+- After updating a PR, add a comment to the GitHub thread mentioning that it
+  is ready for another review. GitHub only notifies maintainers of the
+  changes when you post a comment, so if you don't, your PR will likely be
+  neglected by accident!
 
-There is no bot for other Zulip repositories
-([`zulip/zulip-flutter`](https://github.com/zulip/zulip-flutter/), etc.). If
-you are interested in claiming an issue in one of these repositories, simply
-post a comment on the issue thread saying that you've started work on the
-issue and would like to claim it. In your comment, describe what part of the
-code you're modifying and how you plan to approach the problem, based on
-what you learned in steps 1–4 [above](#picking-an-issue-to-work-on).
+### And beyond
 
-There is no need to @-mention the issue creator in your comment. There is
-also no need to post the same information in multiple places, for example in
-a chat thread in addition to the GitHub issue.
-
-Please follow the same guidelines as described above: find an issue labeled
-"help wanted", and only pick up one issue at a time to start with.
-
-## Getting help
-
-You may have questions as you work on your pull request. For example, you might
-not be sure about some details of what's required, or have questions about your
-implementation approach. Zulip's maintainers are happy to answer thoughtfully
-posed questions, and discuss any difficulties that might arise as you work on
-your PR.
-
-If you haven't done so yet, now is the time to join the [Zulip development
-community](https://zulip.com/development-community/). If you'd like, introduce
-yourself in the [#new
-members](https://chat.zulip.org/#narrow/channel/95-new-members) channel, using
-your name as the [topic](https://zulip.com/help/introduction-to-topics).
-
-You can get help in public channels in the community:
-
-1. **Review** the [Zulip development community
-   guidelines](https://zulip.com/development-community/#community-norms).
-
-1. **Decide where to post.** If there is a discussion thread linked from the
-   issue you're working on, that's usually the best place to post any
-   clarification questions about the issue. Otherwise, follow [these
-   guidelines](https://zulip.com/development-community/#where-do-i-send-my-message)
-   to figure out where to post your question. Don’t stress too much about
-   picking the right place if you’re not sure, as moderators can [move your
-   question thread to a different
-   channel](https://zulip.com/help/move-content-to-another-channel) if needed.
-
-1. **Write** up your question, being sure to follow our [guide on asking great
-   questions](https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html).
-   The guide explains what you need to do make sure that folks will be able to
-   help you out, and that you're making good use of maintainers' limited time.
-
-1. **Review** your message before you send it. Will your question make sense to
-   someone who is familiar with Zulip, but might not have the details of what
-   you are working on fresh in mind?
-
-Well-posed questions will generally get a response within 1-2 business days.
-There is no need to @-mention anyone when you ask a question, as maintainers
-keep a close eye on all the ongoing discussions.
-
-## What makes a great Zulip contributor?
-
-As you're working on your first code contribution, here are some best practices
-to keep in mind.
-
-- [Asking great questions][great-questions]. It's very hard to answer a general
-  question like, "How do I do this issue?" When asking for help, explain your
-  current understanding, including what you've done or tried so far and where
-  you got stuck. Post tracebacks or other error messages if appropriate. For
-  more advice, check out [our guide][great-questions]!
-- Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
-- Submitting carefully tested code. See our [detailed guide on how to review
-  code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
-  (yours or someone else's).
-- Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-- Working to [make your pull requests easy to
-  review](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html).
-- Clearly describing what you have implemented and why. For example, if your
-  implementation differs from the issue description in some way or is a partial
-  step towards the requirements described in the issue, be sure to call
-  out those differences.
-- Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-- Being helpful and friendly on the [Zulip community
-  server](https://zulip.com/development-community/).
-
-[great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
-
-## Submitting a pull request
-
-See the [guide on submitting a pull
-request](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html)
-for detailed instructions on how to present your proposed changes to Zulip.
-
-The [pull request review process
-guide](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-explains the stages of review your PR will go through, and offers guidance on
-how to help the review process move forward.
-
-It's OK if your first issue takes you a while; that's normal! You'll be able to
-work a lot faster as you build experience.
-
-## Beyond the first issue
-
-To find a second issue to work on, we recommend looking through issues with the same
+A great place to look for a second issue is to look for issues with the same
 `area:` label as the last issue you resolved. You'll be able to reuse the
 work you did learning how that part of the codebase works. Also, the path to
 becoming a core developer often involves taking ownership of one of these area
 labels.
 
-## Common questions
+## What makes a great Zulip contributor?
 
-- **What if somebody is already working on the issue I want to claim?** There
-  are lots of issues to work on (likely
-  [hundreds](https://github.com/zulip/zulip/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22%20no%3Aassignee)
-  in the server repository)! If somebody else is actively working on the issue,
-  you can find a different one, or help with reviewing their work.
+Zulip has a lot of experience working with new contributors. In our
+experience, these are the best predictors of success:
 
-- **What if it looks like the person who's assigned an issue is no longer
-  working on it?** Post a comment on the issue, e.g., "Hi @ someone! Are you
-  still working on this one? I'd like to pick it up if not." You can pick up the
-  issue if they say they don't plan to work on it more.
+- Posting good questions. This generally means explaining your current
+  understanding, saying what you've done or tried so far, and including
+  tracebacks or other error messages if appropriate.
+- Learning and practicing
+  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
+- Submitting carefully tested code. This generally means checking your work
+  through a combination of automated tests and manually clicking around the
+  UI trying to find bugs in your work. See
+  [things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
+  for additional ideas.
+- Posting
+  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
+  for frontend changes.
+- Being responsive to feedback on pull requests. This means incorporating or
+  responding to all suggested changes, and leaving a note if you won't be
+  able to address things within a few days.
+- Being helpful and friendly on chat.zulip.org.
 
-  - **What if I don't get a response?** If you don't get a reply within 2-3
-    days, go ahead and post a comment that you are working on the issue, and
-    submit a pull request. If the original assignee ends up submitting a pull
-    request first, no worries! You can help by providing feedback on their work,
-    or submit your own PR if you think a different approach is needed (as
-    described above).
+These are also the main criteria we use to select candidates for all
+of our outreach programs.
 
-- **What if there is already a pull request for the issue I want to work on?**
-  See our [guide on continuing unfinished
-  work](https://zulip.readthedocs.io/en/latest/contributing/continuing-unfinished-work.html).
+## Reporting issues
 
-- **What if somebody else claims an issue while I'm figuring out whether or not to
-  work on it?** No worries! You can contribute by providing feedback on
-  their pull request. If you've made good progress in understanding part of the
-  codebase, you can also find another "help wanted" issue in the same area to
-  work on.
+If you find an easily reproducible bug and/or are experienced in reporting
+bugs, feel free to just open an issue on the relevant project on GitHub.
 
-- **Can I work on an old issue?** Of course! Open issues marked as “help wanted”
-  are generally eligible to be worked on. If you find that the context around
-  the issue has changed (e.g., the UI looks different), do your best to apply
-  the current patterns, and comment on any differences from the spec in your PR
-  description.
+If you have a feature request or are not yet sure what the underlying bug
+is, the best place to post issues is
+[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
+[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
+[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
+[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
+This allows us to interactively figure out what is going on, let you know if
+a similar issue has already been opened, and collect any other information
+we need. Choose a 2-4 word topic that describes the issue, explain the issue
+and how to reproduce it if known, your browser/OS if relevant, and a
+[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
+if appropriate.
 
-  If picking up a bug, start by checking if you can replicate it. If it no longer
-  replicates, post a comment on the issue explaining how you tested the
-  behavior, and what you saw, with screenshots as appropriate. And if you _can_
-  replicate it, fixing it is great!
+**Reporting security issues**. Please do not report security issues
+publicly, including on public streams on chat.zulip.org. You can
+email security@zulip.com. We create a CVE for every security
+issue in our released software.
 
-  If you're starting a major project where the issue was filed more than a
-  couple of years ago, it's a good idea to post to the development community
-  discussion thread for that issue to check if the thinking around it has
-  changed.
+## User feedback
 
-- **Can I come up with my own feature idea and work on it?** We welcome
-  suggestions of features or other improvements that you feel would be valuable. If you
-  have a new feature you'd like to add, you can start a conversation [in our
-  development community](https://zulip.com/development-community/#where-do-i-send-my-message)
-  explaining the feature idea and the problem that you're hoping to solve.
-- **I'm waiting for the next round of review on my PR. Can I pick up
-  another issue in the meantime?** Someone's first Zulip PR often
-  requires quite a bit of iteration, so please [make sure your pull
-  request is reviewable][reviewable-pull-requests] and go through at
-  least one round of feedback from others before picking up a second
-  issue. After that, sure! If
-  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
-  claim an issue, you can post a comment describing the status of your
-  other work on the issue you're interested in (including links to all open
-  PRs), and asking for the issue to be assigned to you. Note that addressing
-  feedback on in-progress PRs should always take priority over starting a new
-  PR.
-- **I think my PR is done, but it hasn't been merged yet. What's going on?**
-  1. **Double-check that you have addressed all the feedback**, including any comments
-     on [Git commit
-     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html),
-     and that automated tests are passing.
-  2. If all the feedback has been addressed, did you [leave a
-     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
-     explaining that you have done so and **requesting another review**? If not,
-     it may not be clear to project maintainers or reviewers that your PR is
-     ready for another look.
-  3. There may be a pause between initial rounds of review for your PR and final
-     review by project maintainers. This is normal, and we encourage you to **work
-     on other issues** while you wait.
-  4. If you think the PR is ready and haven't seen any updates for a couple
-     of weeks, it can be helpful to **leave another comment**. Summarize the
-     overall state of the review process and your work, and indicate that you
-     are waiting for a review.
-  5. Finally, **Zulip project maintainers are people too**! They may be busy
-     with other work, and sometimes they might even take a vacation. ;) It can
-     occasionally take a few weeks for a PR in the final stages of the review
-     process to be merged.
+Nearly every feature we develop starts with a user request. If you are part
+of a group that is either using or considering using Zulip, we would love to
+hear about your experience with the product. If you're not sure what to
+write, here are some questions we're always very curious to know the answer
+to:
 
-[reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
+- Evaluation: What is the process by which your organization chose or will
+  choose a group chat product?
+- Pros and cons: What are the pros and cons of Zulip for your organization,
+  and the pros and cons of other products you are evaluating?
+- Features: What are the features that are most important for your
+  organization? In the best-case scenario, what would your chat solution do
+  for you?
+- Onboarding: If you remember it, what was your impression during your first
+  few minutes of using Zulip? What did you notice, and how did you feel? Was
+  there anything that stood out to you as confusing, or broken, or great?
+- Organization: What does your organization do? How big is the organization?
+  A link to your organization's website?
 
 ## Outreach programs
 
-Zulip regularly participates in [Google Summer of Code
-(GSoC)](https://developers.google.com/open-source/gsoc/) and
-[Outreachy](https://www.outreachy.org/). We have been a GSoC mentoring
-organization since 2016, and we accept 15-20 GSoC participants each summer. In
-the past, we’ve also participated in [Google
-Code-In](https://developers.google.com/open-source/gci/), and hosted summer
-interns from Harvard, MIT, and Stanford.
+Zulip participates in [Google Summer of Code
+(GSoC)](https://developers.google.com/open-source/gsoc/) every year.
+In the past, we've also participated in
+[Outreachy](https://www.outreachy.org/), [Google
+Code-In](https://developers.google.com/open-source/gci/), and hosted
+summer interns from Harvard, MIT, and Stanford.
 
-Check out our [outreach programs
-overview](https://zulip.readthedocs.io/en/latest/outreach/overview.html) to learn
-more about participating in an outreach program with Zulip. Most of our program
-participants end up sticking around the project long-term, and many have become
-core team members, maintaining important parts of the project. We hope you
-apply!
+While each third-party program has its own rules and requirements, the
+Zulip community's approaches all of these programs with these ideas in
+mind:
+
+- We try to make the application process as valuable for the applicant as
+  possible. Expect high-quality code reviews, a supportive community, and
+  publicly viewable patches you can link to from your resume, regardless of
+  whether you are selected.
+- To apply, you'll have to submit at least one pull request to a Zulip
+  repository. Most students accepted to one of our programs have
+  several merged pull requests (including at least one larger PR) by
+  the time of the application deadline.
+- The main criteria we use is quality of your best contributions, and
+  the bullets listed at
+  [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
+  Because we focus on evaluating your best work, it doesn't hurt your
+  application to makes mistakes in your first few PRs as long as your
+  work improves.
+
+Most of our outreach program participants end up sticking around the
+project long-term, and many have become core team members, maintaining
+important parts of the project. We hope you apply!
+
+### Google Summer of Code
+
+The largest outreach program Zulip participates in is GSoC (14
+students in 2017; 11 in 2018; 17 in 2019; 18 in 2020). While we don't control how
+many slots Google allocates to Zulip, we hope to mentor a similar
+number of students in future summers.
+
+If you're reading this well before the application deadline and want
+to make your application strong, we recommend getting involved in the
+community and fixing issues in Zulip now. Having good contributions
+and building a reputation for doing good work is the best way to have
+a strong application. About half of Zulip's GSoC students for Summer
+2017 had made significant contributions to the project by February
+2017, and about half had not. Our
+[GSoC project ideas page][gsoc-guide] has lots more details on how
+Zulip does GSoC, as well as project ideas (though the project idea
+list is maintained only during the GSoC application period, so if
+you're looking at some other time of year, the project list is likely
+out-of-date).
+
+We also have in some past years run a Zulip Summer of Code (ZSoC)
+program for students who we didn't have enough slots to accept for
+GSoC but were able to find funding for. Student expectations are the
+same as with GSoC, and it has no separate application process; your
+GSoC application is your ZSoC application. If we'd like to select you
+for ZSoC, we'll contact you when the GSoC results are announced.
+
+[gsoc-guide]: https://zulip.readthedocs.io/en/latest/contributing/gsoc-ideas.html
+[gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
+
+## Zulip outreach
+
+**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
+perception of projects like Zulip. We've collected a few sites below
+where we know Zulip has been discussed. Doing everything in the following
+list typically takes about 15 minutes.
+
+- Star us on GitHub. There are four main repositories:
+  [server/web](https://github.com/zulip/zulip),
+  [mobile](https://github.com/zulip/zulip-mobile),
+  [desktop](https://github.com/zulip/zulip-desktop), and
+  [Python API](https://github.com/zulip/python-zulip-api).
+- [Follow us](https://twitter.com/zulip) on Twitter.
+
+For both of the following, you'll need to make an account on the site if you
+don't already have one.
+
+- [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
+  AlternativeTo. We recommend upvoting a couple of other products you like
+  as well, both to give back to their community, and since single-upvote
+  accounts are generally given less weight. You can also
+  [upvote Zulip](https://alternativeto.net/software/slack/) on their page
+  for Slack.
+- [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
+  it, and upvote the reasons why people like Zulip that you find most
+  compelling. Again, we recommend adding a few other products that you like
+  as well.
+
+We have a doc with more detailed instructions and a few other sites, if you
+have been using Zulip for a while and want to contribute more.
+
+**Blog posts**. Writing a blog post about your experiences with Zulip, or
+about a technical aspect of Zulip can be a great way to spread the word
+about Zulip.
+
+We also occasionally [publish](https://blog.zulip.org/) long-form
+articles related to Zulip. Our posts typically get tens of thousands
+of views, and we always have good ideas for blog posts that we can
+outline but don't have time to write. If you are an experienced writer
+or copyeditor, send us a portfolio; we'd love to talk!

Dockerfile-postgresql

@@ -1,25 +1,15 @@
-# This is a multiarch Dockerfile.  See https://docs.docker.com/desktop/multi-arch/
-#
-# To set up the first time:
-#     docker buildx create --name multiarch --use
-#
-# To build:
-#     docker buildx build --platform linux/amd64,linux/arm64 \
-#       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
+# To build run `docker build -f Dockerfile-postgresql .` from the root of the
+# zulip repo.
 
 # Currently the PostgreSQL images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image cannot currently be upgraded
+# the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
 
-# https://hub.docker.com/r/groonga/pgroonga/tags
-ARG PGROONGA_VERSION=latest
-ARG POSTGRESQL_VERSION=14
-FROM groonga/pgroonga:$PGROONGA_VERSION-alpine-$POSTGRESQL_VERSION-slim
-
 # Install hunspell, Zulip stop words, and run Zulip database
 # init.
+FROM groonga/pgroonga:latest-alpine-10-slim
 RUN apk add -U --no-cache hunspell-en
-RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix
+RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix 
 COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/local/share/postgresql/tsearch_data/zulip_english.stop
 COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
 COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql

README.md

@@ -1,23 +1,17 @@
 # Zulip overview
 
-[Zulip](https://zulip.com) is an open-source team collaboration tool with unique
-[topic-based threading][why-zulip] that combines the best of email and chat to
-make remote work productive and delightful. Fortune 500 companies, [leading open
-source projects][rust-case-study], and thousands of other organizations use
-Zulip every day. Zulip is the only [modern team chat app][features] that is
-designed for both live and asynchronous conversations.
-
-Zulip is built by a distributed community of developers from all around the
-world, with 74+ people who have each contributed 100+ commits. With
-over 1000 contributors merging over 500 commits a month, Zulip is the
-largest and fastest growing open source team chat project.
-
-Come find us on the [development community chat](https://zulip.com/development-community/)!
+Zulip is a powerful, open source group chat application that combines the
+immediacy of real-time chat with the productivity benefits of threaded
+conversations. Zulip is used by open source projects, Fortune 500 companies,
+large standards bodies, and others who need a real-time chat system that
+allows users to easily process hundreds or thousands of messages a day. With
+over 700 contributors merging over 500 commits a month, Zulip is also the
+largest and fastest growing open source group chat project.
 
 [![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
 [![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
 [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
@@ -26,57 +20,61 @@ Come find us on the [development community chat](https://zulip.com/development-c
 [![GitHub Sponsors](https://img.shields.io/github/sponsors/zulip)](https://github.com/sponsors/zulip)
 
 [mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
-[why-zulip]: https://zulip.com/why-zulip/
-[rust-case-study]: https://zulip.com/case-studies/rust/
-[features]: https://zulip.com/features/
 
 ## Getting started
 
-- **Contributing code**. Check out our [guide for new
-  contributors](https://zulip.readthedocs.io/en/latest/contributing/contributing.html)
-  to get started. We have invested in making Zulip’s code highly
-  readable, thoughtfully tested, and easy to modify. Beyond that, we
-  have written an extraordinary 150K words of documentation for Zulip
-  contributors.
+Click on the appropriate link below. If nothing seems to apply,
+join us on the
+[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
+and tell us what's up!
 
-- **Contributing non-code**. [Report an
-  issue](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#reporting-issues),
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip into your language, or [give us
-  feedback](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html).
-  We'd love to hear from you, whether you've been using Zulip for years, or are just
-  trying it out for the first time.
+You might be interested in:
 
-- **Checking Zulip out**. The best way to see Zulip in action is to drop by the
-  [Zulip community server](https://zulip.com/development-community/). We also
-  recommend reading about Zulip's [unique
-  approach](https://zulip.com/why-zulip/) to organizing conversations.
+- **Contributing code**. Check out our
+  [guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
+  to get started. Zulip prides itself on maintaining a clean and
+  well-tested codebase, and a stock of hundreds of
+  [beginner-friendly issues][beginner-friendly].
 
-- **Running a Zulip server**. Self-host Zulip directly on Ubuntu or Debian
-  Linux, in [Docker](https://github.com/zulip/docker-zulip), or with prebuilt
-  images for [Digital Ocean](https://marketplace.digitalocean.com/apps/zulip) and
-  [Render](https://render.com/docs/deploy-zulip).
-  Learn more about [self-hosting Zulip](https://zulip.com/self-hosting/).
+- **Contributing non-code**.
+  [Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issues),
+  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
+  into your language,
+  [write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
+  for the Zulip blog, or
+  [give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
+  would love to hear from you, even if you're just trying the product out.
 
-- **Using Zulip without setting up a server**. Learn about [Zulip
-  Cloud](https://zulip.com/plans/) hosting options. Zulip sponsors free [Zulip
-  Cloud Standard](https://zulip.com/plans/) for hundreds of worthy
-  organizations, including [fellow open-source
-  projects](https://zulip.com/for/open-source/).
+- **Supporting Zulip**. Advocate for your organization to use Zulip, become a [sponsor](https://github.com/sponsors/zulip), write a
+  review in the mobile app stores, or
+  [upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
+  product comparison sites.
+
+- **Checking Zulip out**. The best way to see Zulip in action is to drop by
+  the
+  [Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
+  also recommend reading Zulip for
+  [open source](https://zulip.com/for/open-source/), Zulip for
+  [companies](https://zulip.com/for/companies/), or Zulip for
+  [working groups and part time communities](https://zulip.com/for/working-groups-and-communities/).
+
+- **Running a Zulip server**. Use a preconfigured [DigitalOcean droplet](https://marketplace.digitalocean.com/apps/zulip),
+  [install Zulip](https://zulip.readthedocs.io/en/stable/production/install.html)
+  directly, or use Zulip's
+  experimental [Docker image](https://zulip.readthedocs.io/en/latest/production/deployment.html#zulip-in-docker).
+  Commercial support is available; see <https://zulip.com/plans> for details.
+
+- **Using Zulip without setting up a server**. <https://zulip.com>
+  offers free and commercial hosting, including providing our paid
+  plan for free to fellow open source projects.
 
 - **Participating in [outreach
-  programs](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#outreach-programs)**
-  like [Google Summer of Code](https://developers.google.com/open-source/gsoc/)
-  and [Outreachy](https://www.outreachy.org/).
-
-- **Supporting Zulip**. Advocate for your organization to use Zulip, become a
-  [sponsor](https://github.com/sponsors/zulip), write a review in the mobile app
-  stores, or [help others find
-  Zulip](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#help-others-find-zulip).
-
-You may also be interested in reading our [blog](https://blog.zulip.org/), and
-following us on [Twitter](https://twitter.com/zulip) and
-[LinkedIn](https://www.linkedin.com/company/zulip-project/).
+  programs](https://zulip.readthedocs.io/en/latest/overview/contributing.html#outreach-programs)**
+  like Google Summer of Code.
 
+You may also be interested in reading our [blog](https://blog.zulip.org/) or
+following us on [Twitter](https://twitter.com/zulip).
 Zulip is distributed under the
 [Apache 2.0](https://github.com/zulip/zulip/blob/main/LICENSE) license.
+
+[beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22

SECURITY.md

@@ -1,5 +1,8 @@
 # Security policy
 
+Security announcements are sent to zulip-announce@googlegroups.com,
+so you should subscribe if you are running Zulip in production.
+
 ## Reporting a vulnerability
 
 We love responsible reports of (potential) security issues in Zulip,
@@ -14,13 +17,6 @@ in our release notes when we publish the fix.
 Our [security model][security-model] document may be a helpful
 resource.
 
-## Security announcements
-
-We send security announcements to our [announcement mailing
-list](https://groups.google.com/g/zulip-announce). If you are running
-Zulip in production, you should subscribe, by clicking "Join group" at
-the top of that page.
-
 ## Supported versions
 
 Zulip provides security support for the latest major release, in the
@@ -29,9 +25,8 @@ form of minor security/maintenance releases.
 We work hard to make [upgrades][upgrades] reliable, so that there's no
 reason to run older major releases.
 
-See also our documentation on the [Zulip release
-lifecycle][release-lifecycle].
+See also our documentation on the [Zulip release lifecycle][release-lifecycle]
 
 [security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
-[upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
-[release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html
+[upgrades]: https://zulip.readthedocs.io/en/latest/production/upgrade-or-modify.html#upgrading-to-a-release
+[release-cycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html

Vagrantfile

@@ -1,8 +1,48 @@
 # -*- mode: ruby -*-
 
-Vagrant.require_version ">= 2.2.6"
+VAGRANTFILE_API_VERSION = "2"
+
+if Vagrant::VERSION == "1.8.7"
+  path = `command -v curl`
+  if path.include?("/opt/vagrant/embedded/bin/curl")
+    puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 " \
+         "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the " \
+         "issue before provisioning. See " \
+         "https://github.com/mitchellh/vagrant/issues/7997 " \
+         "for reference."
+    exit
+  end
+end
+
+# Workaround: Vagrant removed the atlas.hashicorp.com to
+# vagrantcloud.com redirect in February 2018. The value of
+# DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
+# atlas.hashicorp.com, which means that removal broke the fetching and
+# updating of boxes (since the old URL doesn't work).  See
+# https://github.com/hashicorp/vagrant/issues/9442
+if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
+  Vagrant::DEFAULT_SERVER_URL.replace("https://vagrantcloud.com")
+end
+
+# Monkey patch https://github.com/hashicorp/vagrant/pull/10879 so we
+# can fall back to another provider if docker is not installed.
+begin
+  require Vagrant.source_root.join("plugins", "providers", "docker", "provider")
+rescue LoadError
+else
+  VagrantPlugins::DockerProvider::Provider.class_eval do
+    method(:usable?).owner == singleton_class or def self.usable?(raise_error = false)
+      VagrantPlugins::DockerProvider::Driver.new.execute("docker", "version")
+      true
+    rescue Vagrant::Errors::CommandUnavailable, VagrantPlugins::DockerProvider::Errors::ExecuteError
+      raise if raise_error
+      return false
+    end
+  end
+end
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-Vagrant.configure("2") do |config|
   # The Zulip development environment runs on 9991 on the guest.
   host_port = 9991
   http_proxy = https_proxy = no_proxy = nil
@@ -15,10 +55,8 @@ Vagrant.configure("2") do |config|
   ubuntu_mirror = ""
   vboxadd_version = nil
 
-  config.vm.box = "bento/ubuntu-22.04"
-
   config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder ".", "/srv/zulip", docker_consistency: "z"
+  config.vm.synced_folder ".", "/srv/zulip"
 
   vagrant_config_file = ENV["HOME"] + "/.zulip-vagrant-config"
   if File.file?(vagrant_config_file)
@@ -63,7 +101,6 @@ Vagrant.configure("2") do |config|
   config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
   # Specify Docker provider before VirtualBox provider so it's preferred.
   config.vm.provider "docker" do |d, override|
-    override.vm.box = nil
     d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
     d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
     if !ubuntu_mirror.empty?
@@ -74,6 +111,7 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provider "virtualbox" do |vb, override|
+    override.vm.box = "hashicorp/bionic64"
     # It's possible we can get away with just 1.5GB; more testing needed
     vb.memory = vm_memory
     vb.cpus = vm_num_cpus
@@ -90,19 +128,77 @@ Vagrant.configure("2") do |config|
   end
 
   config.vm.provider "hyperv" do |h, override|
+    override.vm.box = "bento/ubuntu-18.04"
     h.memory = vm_memory
     h.maxmemory = vm_memory
     h.cpus = vm_num_cpus
   end
 
   config.vm.provider "parallels" do |prl, override|
+    override.vm.box = "bento/ubuntu-18.04"
+    override.vm.box_version = "202005.21.0"
     prl.memory = vm_memory
     prl.cpus = vm_num_cpus
   end
 
+  $provision_script = <<SCRIPT
+set -x
+set -e
+set -o pipefail
+
+# Code should go here, rather than tools/provision, only if it is
+# something that we don't want to happen when running provision in a
+# development environment not using Vagrant.
+
+# Set the Ubuntu mirror
+[ ! '#{ubuntu_mirror}' ] || sudo sed -i 's|http://\\(\\w*\\.\\)*archive\\.ubuntu\\.com/ubuntu/\\? |#{ubuntu_mirror} |' /etc/apt/sources.list
+
+# Set the MOTD on the system to have Zulip instructions
+sudo ln -nsf /srv/zulip/tools/setup/dev-motd /etc/update-motd.d/99-zulip-dev
+sudo rm -f /etc/update-motd.d/10-help-text
+sudo dpkg --purge landscape-client landscape-common ubuntu-release-upgrader-core update-manager-core update-notifier-common ubuntu-server
+sudo dpkg-divert --add --rename /etc/default/motd-news
+sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
+
+# Set default locale, this prevents errors if the user has another locale set.
+if ! grep -q 'LC_ALL=C.UTF-8' /etc/default/locale; then
+    echo "LC_ALL=C.UTF-8" | sudo tee -a /etc/default/locale
+fi
+
+# Set an environment variable, so that we won't print the virtualenv
+# shell warning (it'll be wrong, since the shell is dying anyway)
+export SKIP_VENV_SHELL_WARNING=1
+
+# End `set -x`, so that the end of provision doesn't look like an error
+# message after a successful run.
+set +x
+
+# Check if the zulip directory is writable
+if [ ! -w /srv/zulip ]; then
+    echo "The vagrant user is unable to write to the zulip directory."
+    echo "To fix this, run the following commands on the host machine:"
+    # sudo is required since our uid is not 1000
+    echo '    vagrant halt -f'
+    echo '    rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
+    echo '    sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
+    echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
+    echo "You can resume setting up your vagrant environment by running:"
+    echo "    vagrant up"
+    exit 1
+fi
+# Provision the development environment
+ln -nsf /srv/zulip ~/zulip
+/srv/zulip/tools/provision
+
+# Run any custom provision hooks the user has configured
+if [ -f /srv/zulip/tools/custom_provision ]; then
+    chmod +x /srv/zulip/tools/custom_provision
+    /srv/zulip/tools/custom_provision
+fi
+SCRIPT
+
   config.vm.provision "shell",
     # We want provision to be run with the permissions of the vagrant user.
     privileged: false,
-    path: "tools/setup/vagrant-provision",
-    env: { "UBUNTU_MIRROR" => ubuntu_mirror }
+    inline: $provision_script
 end

analytics/lib/counts.py

@@ -1,15 +1,13 @@
 import logging
 import time
 from collections import OrderedDict, defaultdict
-from collections.abc import Callable, Sequence
-from datetime import datetime, timedelta, timezone
-from typing import TypeAlias, Union
+from datetime import datetime, timedelta
+from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union
 
 from django.conf import settings
-from django.db import connection, models
-from django.utils.timezone import now as timezone_now
+from django.db import connection
+from django.db.models import F
 from psycopg2.sql import SQL, Composable, Identifier, Literal
-from typing_extensions import override
 
 from analytics.models import (
     BaseCount,
@@ -20,26 +18,26 @@ from analytics.models import (
     UserCount,
     installation_epoch,
 )
+from zerver.lib.logging_util import log_to_file
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
-from zerver.models import Message, Realm, Stream, UserActivityInterval, UserProfile
-from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models import (
+    Message,
+    Realm,
+    RealmAuditLog,
+    Stream,
+    UserActivityInterval,
+    UserProfile,
+    models,
+)
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import (
-        RemoteInstallationCount,
-        RemoteRealm,
-        RemoteRealmCount,
-        RemoteZulipServer,
-    )
-
-
-logger = logging.getLogger("zulip.analytics")
+## Logging setup ##
 
+logger = logging.getLogger("zulip.management")
+log_to_file(logger, settings.ANALYTICS_LOG_PATH)
 
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365 * 1000)
 
-
 ## Class definitions ##
 
 
@@ -59,7 +57,7 @@ class CountStat:
         property: str,
         data_collector: "DataCollector",
         frequency: str,
-        interval: timedelta | None = None,
+        interval: Optional[timedelta] = None,
     ) -> None:
         self.property = property
         self.data_collector = data_collector
@@ -72,11 +70,10 @@ class CountStat:
         else:
             self.interval = self.time_increment
 
-    @override
-    def __repr__(self) -> str:
+    def __str__(self) -> str:
         return f"<CountStat: {self.property}>"
 
-    def last_successful_fill(self) -> datetime | None:
+    def last_successful_fill(self) -> Optional[datetime]:
         fillstate = FillState.objects.filter(property=self.property).first()
         if fillstate is None:
             return None
@@ -84,30 +81,9 @@ class CountStat:
             return fillstate.end_time
         return fillstate.end_time - self.time_increment
 
-    def current_month_accumulated_count_for_user(self, user: UserProfile) -> int:
-        now = timezone_now()
-        start_of_month = datetime(now.year, now.month, 1, tzinfo=timezone.utc)
-        if now.month == 12:  # nocoverage
-            start_of_next_month = datetime(now.year + 1, 1, 1, tzinfo=timezone.utc)
-        else:  # nocoverage
-            start_of_next_month = datetime(now.year, now.month + 1, 1, tzinfo=timezone.utc)
-
-        # We just want to check we are not using BaseCount, otherwise all
-        # `output_table` have `objects` property.
-        assert self.data_collector.output_table == UserCount
-        result = self.data_collector.output_table.objects.filter(  # type: ignore[attr-defined] # see above
-            user=user,
-            property=self.property,
-            end_time__gt=start_of_month,
-            end_time__lte=start_of_next_month,
-        ).aggregate(models.Sum("value"))
-
-        total_value = result["value__sum"] or 0
-        return total_value
-
 
 class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: type[BaseCount], frequency: str) -> None:
+    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
         CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 
 
@@ -117,7 +93,7 @@ class DependentCountStat(CountStat):
         property: str,
         data_collector: "DataCollector",
         frequency: str,
-        interval: timedelta | None = None,
+        interval: Optional[timedelta] = None,
         dependencies: Sequence[str] = [],
     ) -> None:
         CountStat.__init__(self, property, data_collector, frequency, interval=interval)
@@ -127,20 +103,19 @@ class DependentCountStat(CountStat):
 class DataCollector:
     def __init__(
         self,
-        output_table: type[BaseCount],
-        pull_function: Callable[[str, datetime, datetime, Realm | None], int] | None,
+        output_table: Type[BaseCount],
+        pull_function: Optional[Callable[[str, datetime, datetime, Optional[Realm]], int]],
     ) -> None:
         self.output_table = output_table
         self.pull_function = pull_function
 
-    def depends_on_realm(self) -> bool:
-        return self.output_table in (UserCount, StreamCount)
-
 
 ## CountStat-level operations ##
 
 
-def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | None = None) -> None:
+def process_count_stat(
+    stat: CountStat, fill_to_time: datetime, realm: Optional[Realm] = None
+) -> None:
     # TODO: The realm argument is not yet supported, in that we don't
     # have a solution for how to update FillState if it is passed.  It
     # exists solely as partial plumbing for when we do fully implement
@@ -182,7 +157,7 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | N
                 return
             fill_to_time = min(fill_to_time, dependency_fill_time)
 
-    currently_filled += stat.time_increment
+    currently_filled = currently_filled + stat.time_increment
     while currently_filled <= fill_to_time:
         logger.info("START %s %s", stat.property, currently_filled)
         start = time.time()
@@ -190,7 +165,7 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | N
         do_fill_count_stat_at_hour(stat, currently_filled, realm)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
-        currently_filled += stat.time_increment
+        currently_filled = currently_filled + stat.time_increment
         logger.info("DONE %s (%dms)", stat.property, (end - start) * 1000)
 
 
@@ -201,9 +176,9 @@ def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int)
 
 
 # We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
-# and is time-zone-aware. It is the caller's responsibility to enforce this!
+# and is timezone aware. It is the caller's responsibility to enforce this!
 def do_fill_count_stat_at_hour(
-    stat: CountStat, end_time: datetime, realm: Realm | None = None
+    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
     start_time = end_time - stat.interval
     if not isinstance(stat, LoggingCountStat):
@@ -222,7 +197,7 @@ def do_fill_count_stat_at_hour(
 def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
     if isinstance(stat, LoggingCountStat):
         InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.depends_on_realm():
+        if stat.data_collector.output_table in [UserCount, StreamCount]:
             RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
     else:
         UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
@@ -232,18 +207,18 @@ def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
 
 
 def do_aggregate_to_summary_table(
-    stat: CountStat, end_time: datetime, realm: Realm | None = None
+    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
     cursor = connection.cursor()
 
     # Aggregate into RealmCount
     output_table = stat.data_collector.output_table
     if realm is not None:
-        realm_clause: Composable = SQL("AND zerver_realm.id = {}").format(Literal(realm.id))
+        realm_clause = SQL("AND zerver_realm.id = {}").format(Literal(realm.id))
     else:
         realm_clause = SQL("")
 
-    if stat.data_collector.depends_on_realm():
+    if output_table in (UserCount, StreamCount):
         realmcount_query = SQL(
             """
             INSERT INTO analytics_realmcount
@@ -321,12 +296,11 @@ def do_aggregate_to_summary_table(
 
 ## Utility functions called from outside counts.py ##
 
-
-# called from zerver.actions; should not throw any errors
+# called from zerver/lib/actions.py; should not throw any errors
 def do_increment_logging_stat(
-    model_object_for_bucket: Union[Realm, UserProfile, Stream, "RemoteRealm", "RemoteZulipServer"],
+    zerver_object: Union[Realm, UserProfile, Stream],
     stat: CountStat,
-    subgroup: str | int | bool | None,
+    subgroup: Optional[Union[str, int, bool]],
     event_time: datetime,
     increment: int = 1,
 ) -> None:
@@ -334,100 +308,28 @@ def do_increment_logging_stat(
         return
 
     table = stat.data_collector.output_table
-    id_args: dict[str, int | None] = {}
-    conflict_args: list[str] = []
     if table == RealmCount:
-        assert isinstance(model_object_for_bucket, Realm)
-        id_args = {"realm_id": model_object_for_bucket.id}
-        conflict_args = ["realm_id"]
+        id_args = {"realm": zerver_object}
     elif table == UserCount:
-        assert isinstance(model_object_for_bucket, UserProfile)
-        id_args = {
-            "realm_id": model_object_for_bucket.realm_id,
-            "user_id": model_object_for_bucket.id,
-        }
-        conflict_args = ["user_id"]
-    elif table == StreamCount:
-        assert isinstance(model_object_for_bucket, Stream)
-        id_args = {
-            "realm_id": model_object_for_bucket.realm_id,
-            "stream_id": model_object_for_bucket.id,
-        }
-        conflict_args = ["stream_id"]
-    elif table == RemoteInstallationCount:
-        assert isinstance(model_object_for_bucket, RemoteZulipServer)
-        id_args = {"server_id": model_object_for_bucket.id, "remote_id": None}
-        conflict_args = ["server_id"]
-    elif table == RemoteRealmCount:
-        assert isinstance(model_object_for_bucket, RemoteRealm)
-        # For RemoteRealmCount (e.g. `mobile_pushes_forwarded::day`),
-        # we have no `remote_id` nor `realm_id`, since they are not
-        # imported from the remote server, which is the source of
-        # truth of those two columns.  Their "ON CONFLICT" is thus the
-        # only unique key we have, which is `remote_realm_id`, and not
-        # `server_id` / `realm_id`.
-        id_args = {
-            "server_id": model_object_for_bucket.server_id,
-            "remote_realm_id": model_object_for_bucket.id,
-            "remote_id": None,
-            "realm_id": None,
-        }
-        conflict_args = [
-            "remote_realm_id",
-        ]
-    else:
-        raise AssertionError("Unsupported CountStat output_table")
+        id_args = {"realm": zerver_object.realm, "user": zerver_object}
+    else:  # StreamCount
+        id_args = {"realm": zerver_object.realm, "stream": zerver_object}
 
     if stat.frequency == CountStat.DAY:
         end_time = ceiling_to_day(event_time)
-    elif stat.frequency == CountStat.HOUR:
+    else:  # CountStat.HOUR:
         end_time = ceiling_to_hour(event_time)
-    else:
-        raise AssertionError("Unsupported CountStat frequency")
 
-    is_subgroup: SQL = SQL("NULL")
-    if subgroup is not None:
-        is_subgroup = SQL("NOT NULL")
-        # For backwards consistency, we cast the subgroup to a string
-        # in Python; this emulates the behaviour of `get_or_create`,
-        # which was previously used in this function, and performed
-        # this cast because the `subgroup` column is defined as a
-        # `CharField`.  Omitting this explicit cast causes a subgroup
-        # of the boolean False to be passed as the PostgreSQL false,
-        # which it stringifies as the lower-case `'false'`, not the
-        # initial-case `'False'` if Python stringifies it.
-        #
-        # Other parts of the system (e.g. count_message_by_user_query)
-        # already use PostgreSQL to cast bools to strings, resulting
-        # in `subgroup` values of lower-case `'false'` -- for example
-        # in `messages_sent:is_bot:hour`.  Fixing this inconsistency
-        # via a migration is complicated by these records being
-        # exchanged over the wire from remote servers.
-        subgroup = str(subgroup)
-        conflict_args.append("subgroup")
-
-    id_column_names = SQL(", ").join(map(Identifier, id_args.keys()))
-    id_values = SQL(", ").join(map(Literal, id_args.values()))
-    conflict_columns = SQL(", ").join(map(Identifier, conflict_args))
-
-    sql_query = SQL(
-        """
-        INSERT INTO {table_name}(property, subgroup, end_time, value, {id_column_names})
-        VALUES (%s, %s, %s, %s, {id_values})
-        ON CONFLICT (property, end_time, {conflict_columns})
-        WHERE subgroup IS {is_subgroup}
-        DO UPDATE SET
-            value = {table_name}.value + EXCLUDED.value
-        """
-    ).format(
-        table_name=Identifier(table._meta.db_table),
-        id_column_names=id_column_names,
-        id_values=id_values,
-        conflict_columns=conflict_columns,
-        is_subgroup=is_subgroup,
+    row, created = table.objects.get_or_create(
+        property=stat.property,
+        subgroup=subgroup,
+        end_time=end_time,
+        defaults={"value": increment},
+        **id_args,
     )
-    with connection.cursor() as cursor:
-        cursor.execute(sql_query, [stat.property, subgroup, end_time, increment])
+    if not created:
+        row.value = F("value") + increment
+        row.save(update_fields=["value"])
 
 
 def do_drop_all_analytics_tables() -> None:
@@ -448,7 +350,7 @@ def do_drop_single_stat(property: str) -> None:
 
 ## DataCollector-level operations ##
 
-QueryFn: TypeAlias = Callable[[dict[str, Composable]], Composable]
+QueryFn = Callable[[Dict[str, Composable]], Composable]
 
 
 def do_pull_by_sql_query(
@@ -456,11 +358,11 @@ def do_pull_by_sql_query(
     start_time: datetime,
     end_time: datetime,
     query: QueryFn,
-    group_by: tuple[type[models.Model], str] | None,
+    group_by: Optional[Tuple[models.Model, str]],
 ) -> int:
     if group_by is None:
-        subgroup: Composable = SQL("NULL")
-        group_by_clause: Composable = SQL("")
+        subgroup = SQL("NULL")
+        group_by_clause = SQL("")
     else:
         subgroup = Identifier(group_by[0]._meta.db_table, group_by[1])
         group_by_clause = SQL(", {}").format(subgroup)
@@ -490,12 +392,12 @@ def do_pull_by_sql_query(
 
 
 def sql_data_collector(
-    output_table: type[BaseCount],
+    output_table: Type[BaseCount],
     query: QueryFn,
-    group_by: tuple[type[models.Model], str] | None,
+    group_by: Optional[Tuple[models.Model, str]],
 ) -> DataCollector:
     def pull_function(
-        property: str, start_time: datetime, end_time: datetime, realm: Realm | None = None
+        property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
     ) -> int:
         # The pull function type needs to accept a Realm argument
         # because the 'minutes_active::day' CountStat uses
@@ -508,42 +410,8 @@ def sql_data_collector(
     return DataCollector(output_table, pull_function)
 
 
-def count_upload_space_used_by_realm_query(realm: Realm | None) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_attachment.realm_id = {} AND").format(Literal(realm.id))
-
-    # Note: This query currently has to go through the entire table,
-    # summing all the sizes of attachments for every realm. This can be improved
-    # by having a query which looks at the latest CountStat for each realm,
-    # and sums it with only the new attachments.
-    # There'd be additional complexity added by the fact that attachments can
-    # also be deleted. Partially this can be accounted for by subtracting
-    # ArchivedAttachment sizes, but there's still the issue of attachments
-    # which can be directly deleted via the API.
-
-    return lambda kwargs: SQL(
-        """
-            INSERT INTO analytics_realmcount (realm_id, property, end_time, value)
-            SELECT
-                zerver_attachment.realm_id,
-                %(property)s,
-                %(time_end)s,
-                COALESCE(SUM(zerver_attachment.size), 0)
-            FROM
-                zerver_attachment
-            WHERE
-                {realm_clause}
-                zerver_attachment.create_time < %(time_end)s
-            GROUP BY
-                zerver_attachment.realm_id
-        """
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
 def do_pull_minutes_active(
-    property: str, start_time: datetime, end_time: datetime, realm: Realm | None = None
+    property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
 ) -> int:
     user_activity_intervals = (
         UserActivityInterval.objects.filter(
@@ -556,7 +424,7 @@ def do_pull_minutes_active(
         .values_list("user_profile_id", "user_profile__realm_id", "start", "end")
     )
 
-    seconds_active: dict[tuple[int, int], float] = defaultdict(float)
+    seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
     for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
         if realm is None or realm.id == realm_id:
             start = max(start_time, interval_start)
@@ -578,17 +446,11 @@ def do_pull_minutes_active(
     return len(rows)
 
 
-def count_message_by_user_query(realm: Realm | None) -> QueryFn:
+def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -611,17 +473,11 @@ def count_message_by_user_query(realm: Realm | None) -> QueryFn:
 
 
 # Note: ignores the group_by / group_by_clause.
-def count_message_type_by_user_query(realm: Realm | None) -> QueryFn:
+def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -666,14 +522,11 @@ def count_message_type_by_user_query(realm: Realm | None) -> QueryFn:
 # use this also subgroup on UserProfile.is_bot. If in the future there is a
 # stat that counts messages by stream and doesn't need the UserProfile
 # table, consider writing a new query for efficiency.
-def count_message_by_stream_query(realm: Realm | None) -> QueryFn:
+def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
-        realm_clause = SQL(
-            "zerver_stream.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_streamcount
@@ -701,53 +554,81 @@ def count_message_by_stream_query(realm: Realm | None) -> QueryFn:
     ).format(**kwargs, realm_clause=realm_clause)
 
 
-# Hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the
-# same event_time and event_type in [AuditLogEventType.USER_CREATED,
-# USER_DEACTIVATED, etc].  In particular, it's important to ensure
-# that migrations don't cause that to happen.
-def check_realmauditlog_by_user_query(realm: Realm | None) -> QueryFn:
+# Hardcodes the query needed by active_users:is_bot:day, since that is
+# currently the only stat that uses this.
+def count_user_by_realm_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
-        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_userprofile.realm_id, count(*), %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_userprofile
-    JOIN (
-            SELECT DISTINCT ON (modified_user_id)
-                    modified_user_id, event_type
-            FROM
-                    zerver_realmauditlog
-            WHERE
-                    event_type IN ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
-                    {realm_clause}
-                    event_time < %(time_end)s
-            ORDER BY
-                    modified_user_id,
-                    event_time DESC
-    ) last_user_event ON last_user_event.modified_user_id = zerver_userprofile.id
+        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
+    FROM zerver_realm
+    JOIN zerver_userprofile
+    ON
+        zerver_realm.id = zerver_userprofile.realm_id
     WHERE
-        last_user_event.event_type in ({user_created}, {user_activated}, {user_reactivated})
-    GROUP BY zerver_userprofile.realm_id {group_by_clause}
+        zerver_realm.date_created < %(time_end)s AND
+        zerver_userprofile.date_joined >= %(time_start)s AND
+        zerver_userprofile.date_joined < %(time_end)s AND
+        {realm_clause}
+        zerver_userprofile.is_active = TRUE
+    GROUP BY zerver_realm.id {group_by_clause}
+"""
+    ).format(**kwargs, realm_clause=realm_clause)
+
+
+# Currently hardcodes the query needed for active_users_audit:is_bot:day.
+# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
+# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
+# In particular, it's important to ensure that migrations don't cause that to happen.
+def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
+    if realm is None:
+        realm_clause = SQL("")
+    else:
+        realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
+    return lambda kwargs: SQL(
+        """
+    INSERT INTO analytics_usercount
+        (user_id, realm_id, value, property, subgroup, end_time)
+    SELECT
+        ral1.modified_user_id, ral1.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
+    FROM zerver_realmauditlog ral1
+    JOIN (
+        SELECT modified_user_id, max(event_time) AS max_event_time
+        FROM zerver_realmauditlog
+        WHERE
+            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
+            {realm_clause}
+            event_time < %(time_end)s
+        GROUP BY modified_user_id
+    ) ral2
+    ON
+        ral1.event_time = max_event_time AND
+        ral1.modified_user_id = ral2.modified_user_id
+    JOIN zerver_userprofile
+    ON
+        ral1.modified_user_id = zerver_userprofile.id
+    WHERE
+        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
     """
     ).format(
         **kwargs,
-        user_created=Literal(AuditLogEventType.USER_CREATED),
-        user_activated=Literal(AuditLogEventType.USER_ACTIVATED),
-        user_deactivated=Literal(AuditLogEventType.USER_DEACTIVATED),
-        user_reactivated=Literal(AuditLogEventType.USER_REACTIVATED),
+        user_created=Literal(RealmAuditLog.USER_CREATED),
+        user_activated=Literal(RealmAuditLog.USER_ACTIVATED),
+        user_deactivated=Literal(RealmAuditLog.USER_DEACTIVATED),
+        user_reactivated=Literal(RealmAuditLog.USER_REACTIVATED),
         realm_clause=realm_clause,
     )
 
 
-def check_useractivityinterval_by_user_query(realm: Realm | None) -> QueryFn:
+def check_useractivityinterval_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
         realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
@@ -769,9 +650,9 @@ def check_useractivityinterval_by_user_query(realm: Realm | None) -> QueryFn:
     ).format(**kwargs, realm_clause=realm_clause)
 
 
-def count_realm_active_humans_query(realm: Realm | None) -> QueryFn:
+def count_realm_active_humans_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
-        realm_clause: Composable = SQL("")
+        realm_clause = SQL("")
     else:
         realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
@@ -779,46 +660,29 @@ def count_realm_active_humans_query(realm: Realm | None) -> QueryFn:
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-            active_usercount.realm_id, count(*), %(property)s, NULL, %(time_end)s
+        usercount1.realm_id, count(*), %(property)s, NULL, %(time_end)s
     FROM (
-            SELECT
-                    realm_id,
-                    user_id
-            FROM
-                    analytics_usercount
-            WHERE
-                    property = '15day_actives::day'
-                    {realm_clause}
-                    AND end_time = %(time_end)s
-    ) active_usercount
-    JOIN zerver_userprofile ON active_usercount.user_id = zerver_userprofile.id
-     AND active_usercount.realm_id = zerver_userprofile.realm_id
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = 'active_users_audit:is_bot:day' AND
+            subgroup = 'false' AND
+            {realm_clause}
+            end_time = %(time_end)s
+    ) usercount1
     JOIN (
-            SELECT DISTINCT ON (modified_user_id)
-                    modified_user_id, event_type
-            FROM
-                    zerver_realmauditlog
-            WHERE
-                    event_type IN ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated})
-                    AND event_time < %(time_end)s
-            ORDER BY
-                    modified_user_id,
-                    event_time DESC
-    ) last_user_event ON last_user_event.modified_user_id = active_usercount.user_id
-    WHERE
-            NOT zerver_userprofile.is_bot
-            AND event_type IN ({user_created}, {user_activated}, {user_reactivated})
-    GROUP BY
-            active_usercount.realm_id
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = '15day_actives::day' AND
+            {realm_clause}
+            end_time = %(time_end)s
+    ) usercount2
+    ON
+        usercount1.user_id = usercount2.user_id
+    GROUP BY usercount1.realm_id
 """
-    ).format(
-        **kwargs,
-        user_created=Literal(AuditLogEventType.USER_CREATED),
-        user_activated=Literal(AuditLogEventType.USER_ACTIVATED),
-        user_deactivated=Literal(AuditLogEventType.USER_DEACTIVATED),
-        user_reactivated=Literal(AuditLogEventType.USER_REACTIVATED),
-        realm_clause=realm_clause,
-    )
+    ).format(**kwargs, realm_clause=realm_clause)
 
 
 # Currently unused and untested
@@ -841,7 +705,7 @@ count_stream_by_realm_query = lambda kwargs: SQL(
 ).format(**kwargs)
 
 
-def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
+def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
     ## CountStat declarations ##
 
     count_stats_ = [
@@ -874,22 +738,39 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
             ),
             CountStat.DAY,
         ),
-        # AI credit usage stats for users, in units of $1/10^9, which is safe for
-        # aggregation because we're using bigints for the values.
-        LoggingCountStat("ai_credit_usage::day", UserCount, CountStat.DAY),
-        # Counts the number of active users in the UserProfile.is_active sense.
+        # Number of users stats
+        # Stats that count the number of active users in the UserProfile.is_active sense.
+        # 'active_users_audit:is_bot:day' is the canonical record of which users were
+        # active on which days (in the UserProfile.is_active sense).
         # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
         CountStat(
             "active_users_audit:is_bot:day",
             sql_data_collector(
-                RealmCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
+                UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
             ),
             CountStat.DAY,
         ),
+        # Important note: LoggingCountStat objects aren't passed the
+        # Realm argument, because by nature they have a logging
+        # structure, not a pull-from-database structure, so there's no
+        # way to compute them for a single realm after the fact (the
+        # use case for passing a Realm argument).
+        # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
+        # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
+        # sum sequence of 'active_users_log:is_bot:day', for any realm that
+        # started after the latter stat was introduced.
+        LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
+        # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
+        # approximation, e.g. if a user is deactivated between the end of the
+        # day and when this stat is run, they won't be counted. However, is the
+        # simplest of the three to inspect by hand.
         CountStat(
-            "upload_quota_used_bytes::day",
-            sql_data_collector(RealmCount, count_upload_space_used_by_realm_query(realm), None),
+            "active_users:is_bot:day",
+            sql_data_collector(
+                RealmCount, count_user_by_realm_query(realm), (UserProfile, "is_bot")
+            ),
             CountStat.DAY,
+            interval=TIMEDELTA_MAX,
         ),
         # Messages read stats.  messages_read::hour is the total
         # number of messages read, whereas
@@ -923,16 +804,8 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
         CountStat(
             "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
         ),
-        # Tracks the number of push notifications requested by the server.
-        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
-        LoggingCountStat(
-            "mobile_pushes_sent::day",
-            RealmCount,
-            CountStat.DAY,
-        ),
         # Rate limiting stats
-        # Used to limit the number of invitation emails sent by a realm.
-        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
+        # Used to limit the number of invitation emails sent by a realm
         LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
         # Dependent stats
         # Must come after their dependencies.
@@ -941,83 +814,12 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
             "realm_active_humans::day",
             sql_data_collector(RealmCount, count_realm_active_humans_query(realm), None),
             CountStat.DAY,
-            dependencies=["15day_actives::day"],
+            dependencies=["active_users_audit:is_bot:day", "15day_actives::day"],
         ),
     ]
 
-    if settings.ZILENCER_ENABLED:
-        # See also the remote_installation versions of these in REMOTE_INSTALLATION_COUNT_STATS.
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_received::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_forwarded::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-
     return OrderedDict((stat.property, stat) for stat in count_stats_)
 
 
-# These properties are tracked by the bouncer itself and therefore syncing them
-# from a remote server should not be allowed - or the server would be able to interfere
-# with our data.
-BOUNCER_ONLY_REMOTE_COUNT_STAT_PROPERTIES = [
-    "mobile_pushes_received::day",
-    "mobile_pushes_forwarded::day",
-]
-
-# LoggingCountStats with a daily duration and that are directly stored on
-# the RealmCount table (instead of via aggregation in process_count_stat),
-# can be in a state, after the hourly cron job to update analytics counts,
-# where the logged value will be live-updated later (as the end time for
-# the stat is still in the future). As these logging counts are designed
-# to be used on the self-hosted installation for either debugging or rate
-# limiting, sending these incomplete counts to the bouncer has low value.
-LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER = {
-    "invites_sent::day",
-    "mobile_pushes_sent::day",
-    "active_users_log:is_bot:day",
-    "active_users:is_bot:day",
-}
-
 # To avoid refactoring for now COUNT_STATS can be used as before
 COUNT_STATS = get_count_stats()
-
-REMOTE_INSTALLATION_COUNT_STATS = OrderedDict()
-
-if settings.ZILENCER_ENABLED:
-    # REMOTE_INSTALLATION_COUNT_STATS contains duplicates of the
-    # RemoteRealmCount stats declared above; it is necessary because
-    # pre-8.0 servers do not send the fields required to identify a
-    # RemoteRealm.
-
-    # Tracks the number of push notifications requested to be sent
-    # by a remote server.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_received::day"] = LoggingCountStat(
-        "mobile_pushes_received::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-    # Tracks the number of push notifications successfully sent to
-    # mobile devices, as requested by the remote server. Therefore
-    # this should be less than or equal to mobile_pushes_received -
-    # with potential tiny offsets resulting from a request being
-    # *received* by the bouncer right before midnight, but *sent* to
-    # the mobile device right after midnight. This would cause the
-    # increments to happen to CountStat records for different days.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_forwarded::day"] = LoggingCountStat(
-        "mobile_pushes_forwarded::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-
-ALL_COUNT_STATS = OrderedDict(
-    list(COUNT_STATS.items()) + list(REMOTE_INSTALLATION_COUNT_STATS.items())
-)

analytics/lib/fixtures.py

@@ -1,5 +1,6 @@
 from math import sqrt
-from random import Random
+from random import gauss, random, seed
+from typing import List
 
 from analytics.lib.counts import CountStat
 
@@ -15,7 +16,7 @@ def generate_time_series_data(
     frequency: str = CountStat.DAY,
     partial_sum: bool = False,
     random_seed: int = 26,
-) -> list[int]:
+) -> List[int]:
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
 
@@ -35,8 +36,6 @@ def generate_time_series_data(
     partial_sum -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
-    rng = Random(random_seed)
-
     if frequency == CountStat.HOUR:
         length = days * 24
         seasonality = [non_business_hours_base] * 24 * 7
@@ -45,13 +44,13 @@ def generate_time_series_data(
                 seasonality[24 * day + hour] = business_hours_base
         holidays = []
         for i in range(days):
-            holidays.extend([rng.random() < holiday_rate] * 24)
+            holidays.extend([random() < holiday_rate] * 24)
     elif frequency == CountStat.DAY:
         length = days
         seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
             24 * non_business_hours_base
         ] * 2
-        holidays = [rng.random() < holiday_rate for i in range(days)]
+        holidays = [random() < holiday_rate for i in range(days)]
     else:
         raise AssertionError(f"Unknown frequency: {frequency}")
     if length < 2:
@@ -59,17 +58,20 @@ def generate_time_series_data(
             f"Must be generating at least 2 data points. Currently generating {length}"
         )
     growth_base = growth ** (1.0 / (length - 1))
-    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
+    values_no_noise = [
+        seasonality[i % len(seasonality)] * (growth_base ** i) for i in range(length)
+    ]
 
-    noise_scalars = [rng.gauss(0, 1)]
+    seed(random_seed)
+    noise_scalars = [gauss(0, 1)]
     for i in range(1, length):
         noise_scalars.append(
-            noise_scalars[-1] * autocorrelation + rng.gauss(0, 1) * (1 - autocorrelation)
+            noise_scalars[-1] * autocorrelation + gauss(0, 1) * (1 - autocorrelation)
         )
 
     values = [
         0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
-        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays, strict=False)
+        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)
     ]
     if partial_sum:
         for i in range(1, length):

analytics/lib/time_utils.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+from typing import List, Optional
 
 from analytics.lib.counts import CountStat
 from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
@@ -9,8 +10,8 @@ from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
 def time_range(
-    start: datetime, end: datetime, frequency: str, min_length: int | None
-) -> list[datetime]:
+    start: datetime, end: datetime, frequency: str, min_length: Optional[int]
+) -> List[datetime]:
     verify_UTC(start)
     verify_UTC(end)
     if frequency == CountStat.HOUR:
@@ -29,5 +30,4 @@ def time_range(
     while current >= start:
         times.append(current)
         current -= step
-    times.reverse()
-    return times
+    return list(reversed(times))

analytics/management/commands/check_analytics_state.py

@@ -1,15 +1,14 @@
-from dataclasses import dataclass
+import os
+import time
 from datetime import timedelta
-from typing import Any, Literal
+from typing import Any, Dict
 
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, CountStat
+from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.models import installation_epoch
-from scripts.lib.zulip_tools import atomic_nagios_write
-from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
+from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC
 from zerver.models import Realm
 
 states = {
@@ -20,38 +19,37 @@ states = {
 }
 
 
-@dataclass
-class NagiosResult:
-    status: Literal["ok", "warning", "critical", "unknown"]
-    message: str
-
-
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Checks FillState table.
 
     Run as a cron job that runs every hour."""
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         fill_state = self.get_fill_state()
-        atomic_nagios_write("check-analytics-state", fill_state.status, fill_state.message)
+        status = fill_state["status"]
+        message = fill_state["message"]
 
-    def get_fill_state(self) -> NagiosResult:
+        state_file_path = "/var/lib/nagios_state/check-analytics-state"
+        state_file_tmp = state_file_path + "-tmp"
+
+        with open(state_file_tmp, "w") as f:
+            f.write(f"{int(time.time())}|{status}|{states[status]}|{message}\n")
+        os.rename(state_file_tmp, state_file_path)
+
+    def get_fill_state(self) -> Dict[str, Any]:
         if not Realm.objects.exists():
-            return NagiosResult(status="ok", message="No realms exist, so not checking FillState.")
+            return {"status": 0, "message": "No realms exist, so not checking FillState."}
 
         warning_unfilled_properties = []
         critical_unfilled_properties = []
-        for property, stat in ALL_COUNT_STATS.items():
+        for property, stat in COUNT_STATS.items():
             last_fill = stat.last_successful_fill()
             if last_fill is None:
                 last_fill = installation_epoch()
             try:
                 verify_UTC(last_fill)
-            except TimeZoneNotUTCError:
-                return NagiosResult(
-                    status="critical", message=f"FillState not in UTC for {property}"
-                )
+            except TimezoneNotUTCException:
+                return {"status": 2, "message": f"FillState not in UTC for {property}"}
 
             if stat.frequency == CountStat.DAY:
                 floor_function = floor_to_day
@@ -63,10 +61,10 @@ class Command(ZulipBaseCommand):
                 critical_threshold = timedelta(minutes=150)
 
             if floor_function(last_fill) != last_fill:
-                return NagiosResult(
-                    status="critical",
-                    message=f"FillState not on {stat.frequency} boundary for {property}",
-                )
+                return {
+                    "status": 2,
+                    "message": f"FillState not on {stat.frequency} boundary for {property}",
+                }
 
             time_to_last_fill = timezone_now() - last_fill
             if time_to_last_fill > critical_threshold:
@@ -75,18 +73,18 @@ class Command(ZulipBaseCommand):
                 warning_unfilled_properties.append(property)
 
         if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return NagiosResult(status="ok", message="FillState looks fine.")
+            return {"status": 0, "message": "FillState looks fine."}
         if len(critical_unfilled_properties) == 0:
-            return NagiosResult(
-                status="warning",
-                message="Missed filling {} once.".format(
+            return {
+                "status": 1,
+                "message": "Missed filling {} once.".format(
                     ", ".join(warning_unfilled_properties),
                 ),
-            )
-        return NagiosResult(
-            status="critical",
-            message="Missed filling {} once. Missed filling {} at least twice.".format(
+            }
+        return {
+            "status": 2,
+            "message": "Missed filling {} once. Missed filling {} at least twice.".format(
                 ", ".join(warning_unfilled_properties),
                 ", ".join(critical_unfilled_properties),
             ),
-        )
+        }

analytics/management/commands/clear_analytics_tables.py

@@ -1,21 +1,17 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from django.core.management.base import CommandError
-from typing_extensions import override
+from django.core.management.base import BaseCommand, CommandError
 
 from analytics.lib.counts import do_drop_all_analytics_tables
-from zerver.lib.management import ZulipBaseCommand
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         if options["force"]:
             do_drop_all_analytics_tables()

analytics/management/commands/clear_single_stat.py

@@ -1,25 +1,21 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from django.core.management.base import CommandError
-from typing_extensions import override
+from django.core.management.base import BaseCommand, CommandError
 
-from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
-from zerver.lib.management import ZulipBaseCommand
+from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Actually do it.")
         parser.add_argument("--property", help="The property of the stat to be cleared.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         property = options["property"]
-        if property not in ALL_COUNT_STATS:
+        if property not in COUNT_STATS:
             raise CommandError(f"Invalid property: {property}")
         if not options["force"]:
             raise CommandError("No action taken. Use --force.")

analytics/management/commands/populate_analytics_db.py

@@ -1,10 +1,9 @@
-from collections.abc import Mapping
 from datetime import timedelta
-from typing import Any, TypeAlias
+from typing import Any, Dict, List, Mapping, Optional, Type
+from unittest import mock
 
-from django.core.files.uploadedfile import UploadedFile
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
@@ -17,20 +16,13 @@ from analytics.models import (
     StreamCount,
     UserCount,
 )
-from zerver.actions.create_realm import do_create_realm
-from zerver.actions.users import do_change_user_role
+from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, do_change_user_role, do_create_realm
 from zerver.lib.create_user import create_user
-from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.storage import static_path
-from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
-from zerver.lib.streams import get_default_values_for_stream_permission_group_settings
 from zerver.lib.timestamp import floor_to_day
-from zerver.lib.upload import upload_message_attachment_from_request
-from zerver.models import Client, Realm, RealmAuditLog, Recipient, Stream, Subscription, UserProfile
-from zerver.models.realm_audit_logs import AuditLogEventType
+from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserProfile
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
 
     DAYS_OF_DATA = 100
@@ -46,7 +38,7 @@ class Command(ZulipBaseCommand):
         spikiness: float,
         holiday_rate: float = 0,
         partial_sum: bool = False,
-    ) -> list[int]:
+    ) -> List[int]:
         self.random_seed += 1
         return generate_time_series_data(
             days=self.DAYS_OF_DATA,
@@ -61,7 +53,6 @@ class Command(ZulipBaseCommand):
             random_seed=self.random_seed,
         )
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         # TODO: This should arguably only delete the objects
         # associated with the "analytics" realm.
@@ -86,69 +77,40 @@ class Command(ZulipBaseCommand):
             string_id="analytics", name="Analytics", date_created=installation_time
         )
 
-        shylock = create_user(
-            "shylock@analytics.ds",
-            "Shylock",
-            realm,
-            full_name="Shylock",
-            role=UserProfile.ROLE_REALM_OWNER,
-            force_date_joined=installation_time,
-        )
+        with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time):
+            shylock = create_user(
+                "shylock@analytics.ds",
+                "Shylock",
+                realm,
+                full_name="Shylock",
+                role=UserProfile.ROLE_REALM_OWNER,
+            )
         do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None)
-
-        # Create guest user for set_guest_users_statistic.
-        create_user(
-            "bassanio@analytics.ds",
-            "Bassanio",
-            realm,
-            full_name="Bassanio",
-            role=UserProfile.ROLE_GUEST,
-            force_date_joined=installation_time,
-        )
-
-        stream = Stream.objects.create(
-            name="all",
-            realm=realm,
-            date_created=installation_time,
-            **get_default_values_for_stream_permission_group_settings(realm),
-        )
+        stream = Stream.objects.create(name="all", realm=realm, date_created=installation_time)
         recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
         stream.recipient = recipient
         stream.save(update_fields=["recipient"])
 
         # Subscribe shylock to the stream to avoid invariant failures.
-        Subscription.objects.create(
-            recipient=recipient,
-            user_profile=shylock,
-            is_user_active=shylock.is_active,
-            color=STREAM_ASSIGNMENT_COLORS[0],
-        )
-        RealmAuditLog.objects.create(
-            realm=realm,
-            modified_user=shylock,
-            modified_stream=stream,
-            event_last_message_id=0,
-            event_type=AuditLogEventType.SUBSCRIPTION_CREATED,
-            event_time=installation_time,
-        )
-
-        # Create an attachment in the database for set_storage_space_used_statistic.
-        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
-        with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock)
-
-        FixtureData: TypeAlias = Mapping[str | int | None, list[int]]
+        # TODO: This should use subscribe_users_to_streams from populate_db.
+        subs = [
+            Subscription(
+                recipient=recipient,
+                user_profile=shylock,
+                is_user_active=shylock.is_active,
+                color=STREAM_ASSIGNMENT_COLORS[0],
+            ),
+        ]
+        Subscription.objects.bulk_create(subs)
 
         def insert_fixture_data(
-            stat: CountStat,
-            fixture_data: FixtureData,
-            table: type[BaseCount],
+            stat: CountStat, fixture_data: Mapping[Optional[str], List[int]], table: Type[BaseCount]
         ) -> None:
             end_times = time_range(
-                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
+                last_end_time, last_end_time, stat.frequency, len(list(fixture_data.values())[0])
             )
             if table == InstallationCount:
-                id_args: dict[str, Any] = {}
+                id_args: Dict[str, Any] = {}
             if table == RealmCount:
                 id_args = {"realm": realm}
             if table == UserCount:
@@ -157,7 +119,7 @@ class Command(ZulipBaseCommand):
                 id_args = {"stream": stream, "realm": realm}
 
             for subgroup, values in fixture_data.items():
-                table._default_manager.bulk_create(
+                table.objects.bulk_create(
                     table(
                         property=stat.property,
                         subgroup=subgroup,
@@ -165,16 +127,16 @@ class Command(ZulipBaseCommand):
                         value=value,
                         **id_args,
                     )
-                    for end_time, value in zip(end_times, values, strict=False)
+                    for end_time, value in zip(end_times, values)
                     if value != 0
                 )
 
         stat = COUNT_STATS["1day_actives::day"]
-        realm_data: FixtureData = {
+        realm_data: Mapping[Optional[str], List[int]] = {
             None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
         }
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data: FixtureData = {
+        installation_data: Mapping[Optional[str], List[int]] = {
             None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
         }
         insert_fixture_data(stat, installation_data, InstallationCount)
@@ -224,7 +186,7 @@ class Command(ZulipBaseCommand):
         )
 
         stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        user_data: FixtureData = {
+        user_data: Mapping[Optional[str], List[int]] = {
             "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
         }
         insert_fixture_data(stat, user_data, UserCount)
@@ -272,7 +234,6 @@ class Command(ZulipBaseCommand):
         android, created = Client.objects.get_or_create(name="ZulipAndroid")
         iOS, created = Client.objects.get_or_create(name="ZulipiOS")
         react_native, created = Client.objects.get_or_create(name="ZulipMobile")
-        flutter, created = Client.objects.get_or_create(name="ZulipFlutter")
         API, created = Client.objects.get_or_create(name="API: Python")
         zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
         unused, created = Client.objects.get_or_create(name="unused")
@@ -290,7 +251,6 @@ class Command(ZulipBaseCommand):
             android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
             iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
             react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
             zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
             unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
@@ -302,7 +262,6 @@ class Command(ZulipBaseCommand):
             old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
             android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
             iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
             zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
@@ -320,7 +279,7 @@ class Command(ZulipBaseCommand):
             "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
         }
         insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data: Mapping[int | str | None, list[int]] = {
+        stream_data: Mapping[Optional[str], List[int]] = {
             "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
             "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
         }

analytics/management/commands/stream_stats.py

@@ -0,0 +1,61 @@
+from argparse import ArgumentParser
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
+
+from zerver.models import Message, Realm, Recipient, Stream, Subscription, get_realm
+
+
+class Command(BaseCommand):
+    help = "Generate statistics on the streams for a realm."
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        parser.add_argument(
+            "realms", metavar="<realm>", nargs="*", help="realm to generate statistics for"
+        )
+
+    def handle(self, *args: Any, **options: str) -> None:
+        if options["realms"]:
+            try:
+                realms = [get_realm(string_id) for string_id in options["realms"]]
+            except Realm.DoesNotExist as e:
+                raise CommandError(e)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
+            # private stream count
+            private_count = 0
+            # public stream count
+            public_count = 0
+            for stream in streams:
+                if stream.invite_only:
+                    private_count += 1
+                else:
+                    public_count += 1
+            print("------------")
+            print(realm.string_id, end=" ")
+            print("{:>10} {} public streams and".format("(", public_count), end=" ")
+            print(f"{private_count} private streams )")
+            print("------------")
+            print("{:>25} {:>15} {:>10} {:>12}".format("stream", "subscribers", "messages", "type"))
+
+            for stream in streams:
+                if stream.invite_only:
+                    stream_type = "private"
+                else:
+                    stream_type = "public"
+                print(f"{stream.name:>25}", end=" ")
+                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
+                print(
+                    "{:10}".format(
+                        len(Subscription.objects.filter(recipient=recipient, active=True))
+                    ),
+                    end=" ",
+                )
+                num_messages = len(Message.objects.filter(recipient=recipient))
+                print(f"{num_messages:12}", end=" ")
+                print(f"{stream_type:>15}")
+            print("")

analytics/management/commands/update_analytics_counts.py

@@ -1,32 +1,32 @@
-import hashlib
+import os
 import time
 from argparse import ArgumentParser
 from datetime import timezone
-from typing import Any
+from typing import Any, Dict
 
 from django.conf import settings
+from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
-from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from scripts.lib.zulip_tools import ENDC, WARNING
+from zerver.lib.remote_server import send_analytics_to_remote_server
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Fills Analytics tables.
 
     Run as a cron job that runs every hour."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument(
             "--time",
             "-t",
-            help="Update stat tables from current state to --time. Defaults to the current time.",
+            help="Update stat tables from current state to "
+            "--time. Defaults to the current time.",
             default=timezone_now().isoformat(),
         )
         parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
@@ -37,12 +37,22 @@ class Command(ZulipBaseCommand):
             "--verbose", action="store_true", help="Print timing information to stdout."
         )
 
-    @override
-    @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
-        self.run_update_analytics_counts(options)
+        try:
+            os.mkdir(settings.ANALYTICS_LOCK_DIR)
+        except OSError:
+            print(
+                f"{WARNING}Analytics lock {settings.ANALYTICS_LOCK_DIR} is unavailable;"
+                f" exiting.{ENDC}"
+            )
+            return
 
-    def run_update_analytics_counts(self, options: dict[str, Any]) -> None:
+        try:
+            self.run_update_analytics_counts(options)
+        finally:
+            os.rmdir(settings.ANALYTICS_LOCK_DIR)
+
+    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
         # installation_epoch relies on there being at least one realm; we
         # shouldn't run the analytics code if that condition isn't satisfied
         if not Realm.objects.exists():
@@ -50,20 +60,19 @@ class Command(ZulipBaseCommand):
             return
 
         fill_to_time = parse_datetime(options["time"])
-        assert fill_to_time is not None
         if options["utc"]:
             fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
         if fill_to_time.tzinfo is None:
             raise ValueError(
-                "--time must be time-zone-aware. Maybe you meant to use the --utc option?"
+                "--time must be timezone aware. Maybe you meant to use the --utc option?"
             )
 
         fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
 
         if options["stat"] is not None:
-            stats = [ALL_COUNT_STATS[options["stat"]]]
+            stats = [COUNT_STATS[options["stat"]]]
         else:
-            stats = list(ALL_COUNT_STATS.values())
+            stats = list(COUNT_STATS.values())
 
         logger.info("Starting updating analytics counts through %s", fill_to_time)
         if options["verbose"]:
@@ -82,17 +91,5 @@ class Command(ZulipBaseCommand):
             )
         logger.info("Finished updating analytics counts through %s", fill_to_time)
 
-        if should_send_analytics_data():
-            # Based on the specific value of the setting, the exact details to send
-            # will be decided. However, we proceed just based on this not being falsey.
-
-            # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
-            # that each server will report in at a somewhat consistent time.
-            assert settings.ZULIP_ORG_ID
-            delay = int.from_bytes(
-                hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
-            ) % (60 * 10)
-            logger.info("Sleeping %d seconds before reporting...", delay)
-            time.sleep(delay)
-
-            send_server_data_to_push_bouncer(consider_usage_statistics=True, raise_on_error=True)
+        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
+            send_analytics_to_remote_server()

analytics/migrations/0001_initial.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0030_realm_org_type"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),

analytics/migrations/0001_squashed_0021_alter_fillstate_id.py

@@ -1,224 +0,0 @@
-# Generated by Django 5.0.7 on 2024-08-13 20:16
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    replaces = [
-        ("analytics", "0001_initial"),
-        ("analytics", "0002_remove_huddlecount"),
-        ("analytics", "0003_fillstate"),
-        ("analytics", "0004_add_subgroup"),
-        ("analytics", "0005_alter_field_size"),
-        ("analytics", "0006_add_subgroup_to_unique_constraints"),
-        ("analytics", "0007_remove_interval"),
-        ("analytics", "0008_add_count_indexes"),
-        ("analytics", "0009_remove_messages_to_stream_stat"),
-        ("analytics", "0010_clear_messages_sent_values"),
-        ("analytics", "0011_clear_analytics_tables"),
-        ("analytics", "0012_add_on_delete"),
-        ("analytics", "0013_remove_anomaly"),
-        ("analytics", "0014_remove_fillstate_last_modified"),
-        ("analytics", "0015_clear_duplicate_counts"),
-        ("analytics", "0016_unique_constraint_when_subgroup_null"),
-        ("analytics", "0017_regenerate_partial_indexes"),
-        ("analytics", "0018_remove_usercount_active_users_audit"),
-        ("analytics", "0019_remove_unused_counts"),
-        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
-        ("analytics", "0021_alter_fillstate_id"),
-    ]
-
-    initial = True
-
-    dependencies = [
-        # Needed for foreign keys to core models like Realm.
-        ("zerver", "0001_initial"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="InstallationCount",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                ("property", models.CharField(max_length=32)),
-                ("end_time", models.DateTimeField()),
-                ("value", models.BigIntegerField()),
-                ("subgroup", models.CharField(max_length=16, null=True)),
-            ],
-            options={
-                "unique_together": set(),
-                "constraints": [
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", False)),
-                        fields=("property", "subgroup", "end_time"),
-                        name="unique_installation_count",
-                    ),
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", True)),
-                        fields=("property", "end_time"),
-                        name="unique_installation_count_null_subgroup",
-                    ),
-                ],
-            },
-        ),
-        migrations.CreateModel(
-            name="RealmCount",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
-                    ),
-                ),
-                ("property", models.CharField(max_length=32)),
-                ("end_time", models.DateTimeField()),
-                ("value", models.BigIntegerField()),
-                ("subgroup", models.CharField(max_length=16, null=True)),
-            ],
-            options={
-                "indexes": [
-                    models.Index(
-                        fields=["property", "end_time"],
-                        name="analytics_realmcount_property_end_time_3b60396b_idx",
-                    )
-                ],
-                "unique_together": set(),
-                "constraints": [
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", False)),
-                        fields=("realm", "property", "subgroup", "end_time"),
-                        name="unique_realm_count",
-                    ),
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", True)),
-                        fields=("realm", "property", "end_time"),
-                        name="unique_realm_count_null_subgroup",
-                    ),
-                ],
-            },
-        ),
-        migrations.CreateModel(
-            name="StreamCount",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
-                    ),
-                ),
-                (
-                    "stream",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.stream"
-                    ),
-                ),
-                ("property", models.CharField(max_length=32)),
-                ("end_time", models.DateTimeField()),
-                ("value", models.BigIntegerField()),
-                ("subgroup", models.CharField(max_length=16, null=True)),
-            ],
-            options={
-                "indexes": [
-                    models.Index(
-                        fields=["property", "realm", "end_time"],
-                        name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-                    )
-                ],
-                "unique_together": set(),
-                "constraints": [
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", False)),
-                        fields=("stream", "property", "subgroup", "end_time"),
-                        name="unique_stream_count",
-                    ),
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", True)),
-                        fields=("stream", "property", "end_time"),
-                        name="unique_stream_count_null_subgroup",
-                    ),
-                ],
-            },
-        ),
-        migrations.CreateModel(
-            name="UserCount",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
-                    ),
-                ),
-                ("property", models.CharField(max_length=32)),
-                ("end_time", models.DateTimeField()),
-                ("value", models.BigIntegerField()),
-                ("subgroup", models.CharField(max_length=16, null=True)),
-            ],
-            options={
-                "indexes": [
-                    models.Index(
-                        fields=["property", "realm", "end_time"],
-                        name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-                    )
-                ],
-                "unique_together": set(),
-                "constraints": [
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", False)),
-                        fields=("user", "property", "subgroup", "end_time"),
-                        name="unique_user_count",
-                    ),
-                    models.UniqueConstraint(
-                        condition=models.Q(("subgroup__isnull", True)),
-                        fields=("user", "property", "end_time"),
-                        name="unique_user_count_null_subgroup",
-                    ),
-                ],
-            },
-        ),
-        migrations.CreateModel(
-            name="FillState",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                ("property", models.CharField(max_length=40, unique=True)),
-                ("end_time", models.DateTimeField()),
-                ("state", models.PositiveSmallIntegerField()),
-            ],
-        ),
-    ]

analytics/migrations/0002_remove_huddlecount.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0001_initial"),
     ]

analytics/migrations/0003_fillstate.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0002_remove_huddlecount"),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0003_fillstate"),
     ]

analytics/migrations/0005_alter_field_size.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0004_add_subgroup"),
     ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0005_alter_field_size"),
     ]

analytics/migrations/0007_remove_interval.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0006_add_subgroup_to_unique_constraints"),
     ]

analytics/migrations/0008_add_count_indexes.py

@@ -1,33 +1,25 @@
 # Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0050_userprofile_avatar_version"),
         ("analytics", "0007_remove_interval"),
     ]
 
     operations = [
-        migrations.AddIndex(
-            model_name="realmcount",
-            index=models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="realmcount",
+            index_together={("property", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="streamcount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="streamcount",
+            index_together={("property", "realm", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="usercount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="usercount",
+            index_together={("property", "realm", "end_time")},
         ),
     ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,10 +1,10 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
 def delete_messages_sent_to_stream_stat(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+    apps: StateApps, schema_editor: DatabaseSchemaEditor
 ) -> None:
     UserCount = apps.get_model("analytics", "UserCount")
     StreamCount = apps.get_model("analytics", "StreamCount")
@@ -21,10 +21,11 @@ def delete_messages_sent_to_stream_stat(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0008_add_count_indexes"),
     ]
 
     operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat, elidable=True),
+        migrations.RunPython(delete_messages_sent_to_stream_stat),
     ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,10 +1,10 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
 def clear_message_sent_by_message_type_values(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
+    apps: StateApps, schema_editor: DatabaseSchemaEditor
 ) -> None:
     UserCount = apps.get_model("analytics", "UserCount")
     StreamCount = apps.get_model("analytics", "StreamCount")
@@ -21,8 +21,9 @@ def clear_message_sent_by_message_type_values(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
 
     operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values, elidable=True),
+        migrations.RunPython(clear_message_sent_by_message_type_values),
     ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,9 +1,9 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
-def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
     UserCount = apps.get_model("analytics", "UserCount")
     StreamCount = apps.get_model("analytics", "StreamCount")
     RealmCount = apps.get_model("analytics", "RealmCount")
@@ -18,10 +18,11 @@ def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0010_clear_messages_sent_values"),
     ]
 
     operations = [
-        migrations.RunPython(clear_analytics_tables, elidable=True),
+        migrations.RunPython(clear_analytics_tables),
     ]

analytics/migrations/0012_add_on_delete.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0011_clear_analytics_tables"),
     ]

analytics/migrations/0013_remove_anomaly.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0012_add_on_delete"),
     ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0013_remove_anomaly"),
     ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,10 +1,10 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import DatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 from django.db.models import Count, Sum
 
 
-def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
+def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
     """This is a preparatory migration for our Analytics tables.
 
     The backstory is that Django's unique_together indexes do not properly
@@ -55,12 +55,11 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0014_remove_fillstate_last_modified"),
     ]
 
     operations = [
-        migrations.RunPython(
-            clear_duplicate_counts, reverse_code=migrations.RunPython.noop, elidable=True
-        ),
+        migrations.RunPython(clear_duplicate_counts, reverse_code=migrations.RunPython.noop),
     ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0015_clear_duplicate_counts"),
     ]

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -1,114 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0016_unique_constraint_when_subgroup_null"),
-    ]
-
-    # If the server was installed between 7.0 and 7.4 (or main between
-    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
-    # running 7.5 or 7807bff52635 or higher) are never used, because
-    # they contain an improper cast
-    # (https://code.djangoproject.com/ticket/34840).
-    #
-    # We regenerate the indexes here, by dropping and re-creating
-    # them, so that we know that they are properly formed.
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/migrations/0018_remove_usercount_active_users_audit.py

@@ -1,16 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0017_regenerate_partial_indexes"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            "DELETE FROM analytics_usercount WHERE property = 'active_users_audit:is_bot:day'",
-            elidable=True,
-        )
-    ]

analytics/migrations/0019_remove_unused_counts.py

@@ -1,27 +0,0 @@
-from django.db import migrations
-
-REMOVED_COUNTS = (
-    "active_users_log:is_bot:day",
-    "active_users:is_bot:day",
-)
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0018_remove_usercount_active_users_audit"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            [
-                ("DELETE FROM analytics_realmcount WHERE property IN %s", (REMOVED_COUNTS,)),
-                (
-                    "DELETE FROM analytics_installationcount WHERE property IN %s",
-                    (REMOVED_COUNTS,),
-                ),
-            ],
-            elidable=True,
-        )
-    ]

analytics/migrations/0020_alter_installationcount_id_alter_realmcount_id_and_more.py

@@ -1,40 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    atomic = False
-
-    dependencies = [
-        ("analytics", "0019_remove_unused_counts"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/migrations/0021_alter_fillstate_id.py

@@ -1,17 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="fillstate",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/models.py

@@ -1,30 +1,29 @@
-from datetime import datetime
+import datetime
+from typing import Optional
 
 from django.db import models
 from django.db.models import Q, UniqueConstraint
-from typing_extensions import override
 
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Stream, UserProfile
 
 
 class FillState(models.Model):
-    property = models.CharField(max_length=40, unique=True)
-    end_time = models.DateTimeField()
+    property: str = models.CharField(max_length=40, unique=True)
+    end_time: datetime.datetime = models.DateTimeField()
 
     # Valid states are {DONE, STARTED}
     DONE = 1
     STARTED = 2
-    state = models.PositiveSmallIntegerField()
+    state: int = models.PositiveSmallIntegerField()
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.end_time} {self.state}"
+        return f"<FillState: {self.property} {self.end_time} {self.state}>"
 
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime:
+def installation_epoch() -> datetime.datetime:
     earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
         "date_created__min"
     ]
@@ -35,10 +34,10 @@ class BaseCount(models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)
-    subgroup = models.CharField(max_length=16, null=True)
-    end_time = models.DateTimeField()
-    value = models.BigIntegerField()
+    property: str = models.CharField(max_length=32)
+    subgroup: Optional[str] = models.CharField(max_length=16, null=True)
+    end_time: datetime.datetime = models.DateTimeField()
+    value: int = models.BigIntegerField()
 
     class Meta:
         abstract = True
@@ -60,9 +59,8 @@ class InstallationCount(BaseCount):
             ),
         ]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.subgroup} {self.value}"
+        return f"<InstallationCount: {self.property} {self.subgroup} {self.value}>"
 
 
 class RealmCount(BaseCount):
@@ -82,16 +80,10 @@ class RealmCount(BaseCount):
                 name="unique_realm_count_null_subgroup",
             ),
         ]
-        indexes = [
-            models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            )
-        ]
+        index_together = ["property", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
+        return f"<RealmCount: {self.realm} {self.property} {self.subgroup} {self.value}>"
 
 
 class UserCount(BaseCount):
@@ -114,16 +106,10 @@ class UserCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from users to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
+        return f"<UserCount: {self.user} {self.property} {self.subgroup} {self.value}>"
 
 
 class StreamCount(BaseCount):
@@ -146,13 +132,9 @@ class StreamCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from streams to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"
+        return (
+            f"<StreamCount: {self.stream} {self.property} {self.subgroup} {self.value} {self.id}>"
+        )

analytics/tests/test_fixtures.py

@@ -22,7 +22,7 @@ class TestFixtures(ZulipTestCase):
             frequency=CountStat.HOUR,
         )
         # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assert_length(data, 24)
+        self.assertEqual(len(data), 24)
         # test that growth doesn't affect the first data point
         self.assertEqual(data[0], 2000)
         # test that the last data point is growth times what it otherwise would be

analytics/tests/test_stats_views.py

@@ -1,689 +0,0 @@
-from datetime import datetime, timedelta, timezone
-
-from django.utils.timezone import now as timezone_now
-from typing_extensions import override
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, StreamCount, UserCount
-from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client
-from zerver.models.realms import get_realm
-
-
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
-        self.user = self.example_user("hamlet")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_guest_user_cant_access_stats(self) -> None:
-        self.user = self.example_user("polonius")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-        result = self.client_get("/json/analytics/chart_data")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-    def test_stats_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 404)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-
-class TestGetChartData(ZulipTestCase):
-    @override
-    def setUp(self) -> None:
-        super().setUp()
-        self.realm = get_realm("zulip")
-        self.user = self.example_user("hamlet")
-        self.stream_id = self.get_stream_id(self.get_streams(self.user)[0])
-        self.login_user(self.user)
-        self.end_times_hour = [
-            ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
-        ]
-        self.end_times_day = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
-        ]
-
-    def data(self, i: int) -> list[int]:
-        return [0, 0, i, 0]
-
-    def insert_data(
-        self, stat: CountStat, realm_subgroups: list[str | None], user_subgroups: list[str]
-    ) -> None:
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create(
-            RealmCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
-        UserCount.objects.bulk_create(
-            UserCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=200 + i,
-                realm=self.realm,
-                user=self.user,
-            )
-            for i, subgroup in enumerate(user_subgroups)
-        )
-        StreamCount.objects.bulk_create(
-            StreamCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                stream_id=self.stream_id,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "_1day": self.data(100),
-                    "_15day": self.data(100),
-                    "all_time": self.data(100),
-                },
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_over_time(self) -> None:
-        stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        self.insert_data(stat, ["true", "false"], ["false"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "user": {"bot": self.data(0), "human": self.data(200)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_message_type(self) -> None:
-        stat = COUNT_STATS["messages_sent:message_type:day"]
-        self.insert_data(
-            stat, ["public_stream", "private_message"], ["public_stream", "private_stream"]
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "Public channels": self.data(100),
-                    "Private channels": self.data(0),
-                    "Direct messages": self.data(101),
-                    "Group direct messages": self.data(0),
-                },
-                "user": {
-                    "Public channels": self.data(200),
-                    "Private channels": self.data(201),
-                    "Direct messages": self.data(0),
-                    "Group direct messages": self.data(0),
-                },
-                "display_order": [
-                    "Direct messages",
-                    "Public channels",
-                    "Private channels",
-                    "Group direct messages",
-                ],
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_client(self) -> None:
-        stat = COUNT_STATS["messages_sent:client:day"]
-        client1 = Client.objects.create(name="client 1")
-        client2 = Client.objects.create(name="client 2")
-        client3 = Client.objects.create(name="client 3")
-        client4 = Client.objects.create(name="client 4")
-        self.insert_data(
-            stat,
-            [str(client4.id), str(client3.id), str(client2.id)],
-            [str(client3.id), str(client1.id)],
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "client 4": self.data(100),
-                    "client 3": self.data(101),
-                    "client 2": self.data(102),
-                },
-                "user": {"client 3": self.data(200), "client 1": self.data(201)},
-                "display_order": ["client 1", "client 2", "client 3", "client 4"],
-                "result": "success",
-            },
-        )
-
-    def test_messages_read_over_time(self) -> None:
-        stat = COUNT_STATS["messages_read::hour"]
-        self.insert_data(stat, [None], [])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_read_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"read": self.data(100)},
-                "user": {"read": self.data(0)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_stream(self) -> None:
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
-        self.insert_data(stat, ["true", "false"], [])
-
-        result = self.client_get(
-            f"/json/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-        result = self.api_get(
-            self.example_user("polonius"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        self.assert_json_error(result, "Not allowed for guest users")
-
-        # Verify we correctly forbid access to stats of streams in other realms.
-        result = self.api_get(
-            self.mit_user("sipbtest"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-            subdomain="zephyr",
-        )
-        self.assert_json_error(result, "Invalid channel ID")
-
-    def test_include_empty_subgroups(self) -> None:
-        FillState.objects.create(
-            property="realm_active_humans::day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"_1day": [0], "_15day": [0], "all_time": [0]})
-        self.assertFalse("user" in data)
-
-        FillState.objects.create(
-            property="messages_sent:is_bot:hour",
-            end_time=self.end_times_hour[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"human": [0], "bot": [0]})
-        self.assertEqual(data["user"], {"human": [0], "bot": [0]})
-
-        FillState.objects.create(
-            property="messages_sent:message_type:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["everyone"],
-            {
-                "Public channels": [0],
-                "Private channels": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-        self.assertEqual(
-            data["user"],
-            {
-                "Public channels": [0],
-                "Private channels": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-
-        FillState.objects.create(
-            property="messages_sent:client:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {})
-        self.assertEqual(data["user"], {})
-
-    def test_start_and_end(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[1],
-                "end": end_time_timestamps[2],
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["end_times"], end_time_timestamps[1:3])
-        self.assertEqual(
-            data["everyone"], {"_1day": [0, 100], "_15day": [0, 100], "all_time": [0, 100]}
-        )
-
-        # start later then end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[2],
-                "end": end_time_timestamps[1],
-            },
-        )
-        self.assert_json_error_contains(result, "Start time is later than")
-
-    def test_min_length(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        # test min_length is too short to change anything
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 2}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["end_times"], [datetime_to_timestamp(dt) for dt in self.end_times_day]
-        )
-        self.assertEqual(
-            data["everyone"],
-            {"_1day": self.data(100), "_15day": self.data(100), "all_time": self.data(100)},
-        )
-        # test min_length larger than filled data
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 5}
-        )
-        data = self.assert_json_success(result)
-        end_times = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)
-        ]
-        self.assertEqual(data["end_times"], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(
-            data["everyone"],
-            {
-                "_1day": [0, *self.data(100)],
-                "_15day": [0, *self.data(100)],
-                "all_time": [0, *self.data(100)],
-            },
-        )
-
-    def test_non_existent_chart(self) -> None:
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "does_not_exist"})
-        self.assert_json_error_contains(result, "Unknown chart name")
-
-    def test_analytics_not_running(self) -> None:
-        realm = get_realm("zulip")
-
-        self.assertEqual(FillState.objects.count(), 0)
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(hours=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=5)
-        fill_state = FillState.objects.create(
-            property="messages_sent:is_bot:hour", end_time=end_time, state=FillState.DONE
-        )
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=2)
-        fill_state.end_time = end_time
-        fill_state.save(update_fields=["end_time"])
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/not_existing_realm",
-            {"chart_name": "number_of_humans"},
-        )
-        self.assert_json_error(result, "Invalid organization", 400)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    def test_sort_by_totals(self) -> None:
-        empty: list[int] = []
-        value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
-        self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
-
-    def test_sort_client_labels(self) -> None:
-        data = {
-            "everyone": {"a": [16], "c": [15], "b": [14], "e": [13], "d": [12], "h": [11]},
-            "user": {"a": [6], "b": [5], "d": [4], "e": [3], "f": [2], "g": [1]},
-        }
-        self.assertEqual(sort_client_labels(data), ["a", "b", "c", "d", "e", "f", "g", "h"])
-
-
-class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
-        HOUR = timedelta(hours=1)
-        DAY = timedelta(days=1)
-
-        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
-        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
-        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
-
-        # test start == end
-        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
-        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
-        # test start == end == boundary, and min_length == 0
-        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
-        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
-        # test start and end on different boundaries
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, None),
-            [floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, None),
-            [floor_day, floor_day + DAY],
-        )
-        # test min_length
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, 4),
-            [floor_hour - 2 * HOUR, floor_hour - HOUR, floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, 4),
-            [floor_day - 2 * DAY, floor_day - DAY, floor_day, floor_day + DAY],
-        )
-
-
-class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
-        a = {
-            "desktop app 1.0": [1, 2, 3],
-            "desktop app 2.0": [10, 12, 13],
-            "desktop app 3.0": [21, 22, 23],
-            "website": [1, 2, 3],
-            "ZulipiOS": [1, 2, 3],
-            "ZulipElectron": [2, 5, 7],
-            "ZulipMobile": [1, 2, 3],
-            "ZulipMobile/flutter": [1, 1, 1],
-            "ZulipFlutter": [1, 1, 1],
-            "ZulipPython": [1, 2, 3],
-            "API: Python": [1, 2, 3],
-            "SomethingRandom": [4, 5, 6],
-            "ZulipGitHubWebhook": [7, 7, 9],
-            "ZulipAndroid": [64, 63, 65],
-            "ZulipTerminal": [9, 10, 11],
-        }
-        result = rewrite_client_arrays(a)
-        self.assertEqual(
-            result,
-            {
-                "Old desktop app": [32, 36, 39],
-                "Old iOS app": [1, 2, 3],
-                "Desktop app": [2, 5, 7],
-                "Mobile app (React Native)": [1, 2, 3],
-                "Mobile app beta (Flutter)": [2, 2, 2],
-                "Web app": [1, 2, 3],
-                "Python API": [2, 4, 6],
-                "SomethingRandom": [4, 5, 6],
-                "GitHub webhook": [7, 7, 9],
-                "Old Android app": [64, 63, 65],
-                "Terminal app": [9, 10, 11],
-            },
-        )

analytics/urls.py

@@ -1,38 +1,40 @@
-from django.conf import settings
 from django.conf.urls import include
 from django.urls import path
-from django.urls.resolvers import URLPattern, URLResolver
 
-from analytics.views.stats import (
+from analytics.views import (
+    get_activity,
     get_chart_data,
     get_chart_data_for_installation,
     get_chart_data_for_realm,
-    get_chart_data_for_stream,
+    get_chart_data_for_remote_installation,
+    get_chart_data_for_remote_realm,
+    get_realm_activity,
+    get_user_activity,
     stats,
     stats_for_installation,
     stats_for_realm,
+    stats_for_remote_installation,
+    stats_for_remote_realm,
+    support,
 )
 from zerver.lib.rest import rest_path
 
-i18n_urlpatterns: list[URLPattern | URLResolver] = [
+i18n_urlpatterns = [
     # Server admin (user_profile.is_staff) visible stats pages
+    path("activity", get_activity),
+    path("activity/support", support, name="support"),
+    path("realm_activity/<realm_str>/", get_realm_activity),
+    path("user_activity/<email>/", get_user_activity),
     path("stats/realm/<realm_str>/", stats_for_realm),
     path("stats/installation", stats_for_installation),
+    path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
+    path(
+        "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
+    ),
     # User-visible stats page
     path("stats", stats, name="stats"),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import stats_for_remote_installation, stats_for_remote_realm
-
-    i18n_urlpatterns += [
-        path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
-        path(
-            "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/",
-            stats_for_remote_realm,
-        ),
-    ]
-
 # These endpoints are a part of the API (V1), which uses:
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
@@ -44,28 +46,18 @@ if settings.ZILENCER_ENABLED:
 v1_api_and_json_patterns = [
     # get data for the graphs at /stats
     rest_path("analytics/chart_data", GET=get_chart_data),
-    rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
     rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
     rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/installation",
+        GET=get_chart_data_for_remote_installation,
+    ),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
+        GET=get_chart_data_for_remote_realm,
+    ),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import (
-        get_chart_data_for_remote_installation,
-        get_chart_data_for_remote_realm,
-    )
-
-    v1_api_and_json_patterns += [
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/installation",
-            GET=get_chart_data_for_remote_installation,
-        ),
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
-            GET=get_chart_data_for_remote_realm,
-        ),
-    ]
-
 i18n_urlpatterns += [
     path("api/v1/", include(v1_api_and_json_patterns)),
     path("json/", include(v1_api_and_json_patterns)),

analytics/views/stats.py

@@ -1,655 +0,0 @@
-import logging
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-from typing import Annotated, Any, Optional, TypeAlias, TypeVar, cast
-
-from django.conf import settings
-from django.db.models import QuerySet
-from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
-from django.shortcuts import render
-from django.utils import translation
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-from pydantic import BeforeValidator, Json, NonNegativeInt
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-    installation_epoch,
-)
-from zerver.decorator import (
-    require_non_guest_user,
-    require_server_admin,
-    require_server_admin_api,
-    to_utc_datetime,
-    zulip_login_required,
-)
-from zerver.lib.exceptions import JsonableError
-from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
-from zerver.lib.response import json_success
-from zerver.lib.streams import access_stream_by_id
-from zerver.lib.timestamp import convert_to_UTC
-from zerver.lib.typed_endpoint import PathOnly, typed_endpoint
-from zerver.models import Client, Realm, Stream, UserProfile
-from zerver.models.realms import get_realm
-
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
-
-MAX_TIME_FOR_FULL_ANALYTICS_GENERATION = timedelta(days=1, minutes=30)
-
-
-def is_analytics_ready(realm: Realm) -> bool:
-    return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
-
-
-def render_stats(
-    request: HttpRequest,
-    data_url_suffix: str,
-    realm: Realm | None,
-    *,
-    title: str | None = None,
-    analytics_ready: bool = True,
-) -> HttpResponse:
-    assert request.user.is_authenticated
-
-    if realm is not None:
-        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
-        guest_users = UserProfile.objects.filter(
-            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
-        ).count()
-        space_used = realm.currently_used_upload_space_bytes()
-        if title:
-            pass
-        else:
-            title = realm.name or realm.string_id
-    else:
-        assert title
-        guest_users = None
-        space_used = None
-
-    request_language = get_and_set_request_language(
-        request,
-        request.user.default_language,
-        translation.get_language_from_path(request.path_info),
-    )
-
-    # Sync this with stats_params_schema in base_page_params.ts.
-    page_params = dict(
-        page_type="stats",
-        data_url_suffix=data_url_suffix,
-        upload_space_used=space_used,
-        guest_users=guest_users,
-        translation_data=get_language_translation_data(request_language),
-    )
-
-    return render(
-        request,
-        "analytics/stats.html",
-        context=dict(
-            target_name=title,
-            page_params=page_params,
-            analytics_ready=analytics_ready,
-        ),
-    )
-
-
-@zulip_login_required
-def stats(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    realm = request.user.realm
-    if request.user.is_guest:
-        # TODO: Make @zulip_login_required pass the UserProfile so we
-        # can use @require_member_or_admin
-        raise JsonableError(_("Not allowed for guest users"))
-    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
-
-
-@require_server_admin
-@typed_endpoint
-def stats_for_realm(request: HttpRequest, *, realm_str: PathOnly[str]) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        return HttpResponseNotFound()
-
-    return render_stats(
-        request,
-        f"/realm/{realm_str}",
-        realm,
-        analytics_ready=is_analytics_ready(realm),
-    )
-
-
-@require_server_admin
-@typed_endpoint
-def stats_for_remote_realm(
-    request: HttpRequest, *, remote_server_id: PathOnly[int], remote_realm_id: PathOnly[int]
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/realm/{remote_realm_id}",
-        None,
-        title=f"Realm {remote_realm_id} on server {server.hostname}",
-    )
-
-
-@require_server_admin_api
-@typed_endpoint
-def get_chart_data_for_realm(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    /,
-    *,
-    realm_str: PathOnly[str],
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        raise JsonableError(_("Invalid organization"))
-
-    return do_get_chart_data(
-        request,
-        user_profile,
-        realm=realm,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_non_guest_user
-@typed_endpoint
-def get_chart_data_for_stream(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    *,
-    stream_id: PathOnly[int],
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    stream, ignored_sub = access_stream_by_id(
-        user_profile,
-        stream_id,
-        require_active=True,
-        require_content_access=False,
-    )
-
-    return do_get_chart_data(
-        request,
-        user_profile,
-        stream=stream,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_server_admin_api
-@typed_endpoint
-def get_chart_data_for_remote_realm(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    /,
-    *,
-    remote_server_id: PathOnly[int],
-    remote_realm_id: PathOnly[int],
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return do_get_chart_data(
-        request,
-        user_profile,
-        server=server,
-        remote=True,
-        remote_realm_id=remote_realm_id,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_server_admin
-def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation")
-
-
-@require_server_admin
-def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/installation",
-        None,
-        title=f"remote installation {server.hostname}",
-    )
-
-
-@require_server_admin_api
-@typed_endpoint
-def get_chart_data_for_installation(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    /,
-    *,
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    return do_get_chart_data(
-        request,
-        user_profile,
-        for_installation=True,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_server_admin_api
-@typed_endpoint
-def get_chart_data_for_remote_installation(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    /,
-    *,
-    remote_server_id: PathOnly[int],
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return do_get_chart_data(
-        request,
-        user_profile,
-        for_installation=True,
-        remote=True,
-        server=server,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_non_guest_user
-@typed_endpoint
-def get_chart_data(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    *,
-    chart_name: str,
-    min_length: Json[NonNegativeInt] | None = None,
-    start: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-    end: Annotated[datetime | None, BeforeValidator(to_utc_datetime)] = None,
-) -> HttpResponse:
-    return do_get_chart_data(
-        request,
-        user_profile,
-        chart_name=chart_name,
-        min_length=min_length,
-        start=start,
-        end=end,
-    )
-
-
-@require_non_guest_user
-def do_get_chart_data(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    *,
-    # Common parameters supported by all stats endpoints.
-    chart_name: str,
-    min_length: NonNegativeInt | None = None,
-    start: datetime | None = None,
-    end: datetime | None = None,
-    # The following parameters are only used by wrapping functions for
-    # various contexts; the callers are responsible for validating them.
-    realm: Realm | None = None,
-    for_installation: bool = False,
-    remote: bool = False,
-    remote_realm_id: int | None = None,
-    server: Optional["RemoteZulipServer"] = None,
-    stream: Stream | None = None,
-) -> HttpResponse:
-    TableType: TypeAlias = (
-        type["RemoteInstallationCount"]
-        | type[InstallationCount]
-        | type["RemoteRealmCount"]
-        | type[RealmCount]
-    )
-    if for_installation:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table: TableType = RemoteInstallationCount
-            assert server is not None
-        else:
-            aggregate_table = InstallationCount
-    else:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table = RemoteRealmCount
-            assert server is not None
-            assert remote_realm_id is not None
-        else:
-            aggregate_table = RealmCount
-
-    tables: (
-        tuple[TableType] | tuple[TableType, type[UserCount]] | tuple[TableType, type[StreamCount]]
-    )
-
-    if chart_name == "number_of_humans":
-        stats = [
-            COUNT_STATS["1day_actives::day"],
-            COUNT_STATS["realm_active_humans::day"],
-            COUNT_STATS["active_users_audit:is_bot:day"],
-        ]
-        tables = (aggregate_table,)
-        subgroup_to_label: dict[CountStat, dict[str | None, str]] = {
-            stats[0]: {None: "_1day"},
-            stats[1]: {None: "_15day"},
-            stats[2]: {"false": "all_time"},
-        }
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_over_time":
-        stats = [COUNT_STATS["messages_sent:is_bot:hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_message_type":
-        stats = [COUNT_STATS["messages_sent:message_type:day"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {
-            stats[0]: {
-                "public_stream": _("Public channels"),
-                "private_stream": _("Private channels"),
-                "private_message": _("Direct messages"),
-                "huddle_message": _("Group direct messages"),
-            }
-        }
-        labels_sort_function = lambda data: sort_by_totals(data["everyone"])
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_client":
-        stats = [COUNT_STATS["messages_sent:client:day"]]
-        tables = (aggregate_table, UserCount)
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {
-            stats[0]: {str(id): name for id, name in Client.objects.values_list("id", "name")}
-        }
-        labels_sort_function = sort_client_labels
-        include_empty_subgroups = False
-    elif chart_name == "messages_read_over_time":
-        stats = [COUNT_STATS["messages_read::hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {None: "read"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_stream":
-        if stream is None:
-            raise JsonableError(
-                _("Missing channel for chart: {chart_name}").format(chart_name=chart_name)
-            )
-        stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
-        tables = (aggregate_table, StreamCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    else:
-        raise JsonableError(_("Unknown chart name: {chart_name}").format(chart_name=chart_name))
-
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None:
-        start = convert_to_UTC(start)
-    if end is not None:
-        end = convert_to_UTC(end)
-    if start is not None and end is not None and start > end:
-        raise JsonableError(
-            _("Start time is later than end time. Start: {start}, End: {end}").format(
-                start=start,
-                end=end,
-            )
-        )
-
-    if realm is None:
-        # Note that this value is invalid for Remote tables; be
-        # careful not to access it in those code paths.
-        realm = user_profile.realm
-
-    if remote:
-        # For remote servers, we don't have fillstate data, and thus
-        # should simply use the first and last data points for the
-        # table.
-        assert server is not None
-        assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
-        aggregate_table_remote = cast(
-            type[RemoteInstallationCount] | type[RemoteRealmCount], aggregate_table
-        )  # https://stackoverflow.com/questions/68540528/mypy-assertions-on-the-types-of-types
-        if not aggregate_table_remote.objects.filter(server=server).exists():
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-        if start is None:
-            first = (
-                aggregate_table_remote.objects.filter(server=server).order_by("remote_id").first()
-            )
-            assert first is not None
-            start = first.end_time
-        if end is None:
-            last = aggregate_table_remote.objects.filter(server=server).order_by("remote_id").last()
-            assert last is not None
-            end = last.end_time
-    else:
-        # Otherwise, we can use tables on the current server to
-        # determine a nice range, and some additional validation.
-        if start is None:
-            if for_installation:
-                start = installation_epoch()
-            else:
-                start = realm.date_created
-        if end is None:
-            end = max(
-                stat.last_successful_fill() or datetime.min.replace(tzinfo=timezone.utc)
-                for stat in stats
-            )
-
-        if start > end and (timezone_now() - start > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION):
-            logging.warning(
-                "User from realm %s attempted to access /stats, but the computed "
-                "start time: %s (creation of realm or installation) is later than the computed "
-                "end time: %s (last successful analytics update). Is the "
-                "analytics cron job running?",
-                realm.string_id,
-                start,
-                end,
-            )
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-
-    assert len({stat.frequency for stat in stats}) == 1
-    end_times = time_range(start, end, stats[0].frequency, min_length)
-    data: dict[str, Any] = {
-        "end_times": [int(end_time.timestamp()) for end_time in end_times],
-        "frequency": stats[0].frequency,
-    }
-
-    aggregation_level = {
-        InstallationCount: "everyone",
-        RealmCount: "everyone",
-        UserCount: "user",
-        StreamCount: "everyone",
-    }
-    if settings.ZILENCER_ENABLED:
-        aggregation_level[RemoteInstallationCount] = "everyone"
-        aggregation_level[RemoteRealmCount] = "everyone"
-
-    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
-    id_value = {
-        InstallationCount: -1,
-        RealmCount: realm.id,
-        UserCount: user_profile.id,
-    }
-    if stream is not None:
-        id_value[StreamCount] = stream.id
-
-    if settings.ZILENCER_ENABLED:
-        if server is not None:
-            id_value[RemoteInstallationCount] = server.id
-        # TODO: RemoteRealmCount logic doesn't correctly handle
-        # filtering by server_id as well.
-        if remote_realm_id is not None:
-            id_value[RemoteRealmCount] = remote_realm_id
-
-    for table in tables:
-        data[aggregation_level[table]] = {}
-        for stat in stats:
-            data[aggregation_level[table]].update(
-                get_time_series_by_subgroup(
-                    stat,
-                    table,
-                    id_value[table],
-                    end_times,
-                    subgroup_to_label[stat],
-                    include_empty_subgroups,
-                )
-            )
-
-    if labels_sort_function is not None:
-        data["display_order"] = labels_sort_function(data)
-    else:
-        data["display_order"] = None
-    return json_success(request, data=data)
-
-
-def sort_by_totals(value_arrays: dict[str, list[int]]) -> list[str]:
-    totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
-    return [label for total, label in totals]
-
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data: dict[str, dict[str, list[int]]]) -> list[str]:
-    realm_order = sort_by_totals(data["everyone"])
-    user_order = sort_by_totals(data["user"])
-    label_sort_values: dict[str, float] = {label: i for i, label in enumerate(realm_order)}
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
-
-
-CountT = TypeVar("CountT", bound=BaseCount)
-
-
-def table_filtered_to_id(table: type[CountT], key_id: int) -> QuerySet[CountT]:
-    if table == RealmCount:
-        return table._default_manager.filter(realm_id=key_id)
-    elif table == UserCount:
-        return table._default_manager.filter(user_id=key_id)
-    elif table == StreamCount:
-        return table._default_manager.filter(stream_id=key_id)
-    elif table == InstallationCount:
-        return table._default_manager.all()
-    elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
-        return table._default_manager.filter(server_id=key_id)
-    elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
-        return table._default_manager.filter(realm_id=key_id)
-    else:
-        raise AssertionError(f"Unknown table: {table}")
-
-
-def client_label_map(name: str) -> str:
-    if name == "website":
-        return "Web app"
-    if name.startswith("desktop app"):
-        return "Old desktop app"
-    if name == "ZulipElectron":
-        return "Desktop app"
-    if name == "ZulipTerminal":
-        return "Terminal app"
-    if name == "ZulipAndroid":
-        return "Old Android app"
-    if name == "ZulipiOS":
-        return "Old iOS app"
-    if name == "ZulipMobile":
-        return "Mobile app (React Native)"
-    if name in ["ZulipFlutter", "ZulipMobile/flutter"]:
-        return "Mobile app beta (Flutter)"
-    if name in ["ZulipPython", "API: Python"]:
-        return "Python API"
-    if name.startswith("Zulip") and name.endswith("Webhook"):
-        return name.removeprefix("Zulip").removesuffix("Webhook") + " webhook"
-    return name
-
-
-def rewrite_client_arrays(value_arrays: dict[str, list[int]]) -> dict[str, list[int]]:
-    mapped_arrays: dict[str, list[int]] = {}
-    for label, array in value_arrays.items():
-        mapped_label = client_label_map(label)
-        if mapped_label in mapped_arrays:
-            for i in range(len(array)):
-                mapped_arrays[mapped_label][i] += value_arrays[label][i]
-        else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
-    return mapped_arrays
-
-
-def get_time_series_by_subgroup(
-    stat: CountStat,
-    table: type[BaseCount],
-    key_id: int,
-    end_times: list[datetime],
-    subgroup_to_label: dict[str | None, str],
-    include_empty_subgroups: bool,
-) -> dict[str, list[int]]:
-    queryset = (
-        table_filtered_to_id(table, key_id)
-        .filter(property=stat.property)
-        .values_list("subgroup", "end_time", "value")
-    )
-    value_dicts: dict[str | None, dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
-    for subgroup, end_time, value in queryset:
-        value_dicts[subgroup][end_time] = value
-    value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
-        if (subgroup in value_dicts) or include_empty_subgroups:
-            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
-
-    if stat == COUNT_STATS["messages_sent:client:day"]:
-        # HACK: We rewrite these arrays to collapse the Client objects
-        # with similar names into a single sum, and generally give
-        # them better names
-        return rewrite_client_arrays(value_arrays)
-    return value_arrays

api_docs/api-doc-template.md

@@ -1,31 +0,0 @@
-{generate_api_header(API_ENDPOINT_NAME)}
-
-## Usage examples
-
-{start_tabs}
-
-{generate_code_example(python)|API_ENDPOINT_NAME|example}
-
-{generate_code_example(javascript)|API_ENDPOINT_NAME|example}
-
-{tab|curl}
-
-{generate_code_example(curl)|API_ENDPOINT_NAME|example}
-
-{end_tabs}
-
-## Parameters
-
-{generate_api_arguments_table|zulip.yaml|API_ENDPOINT_NAME}
-
-{generate_parameter_description(API_ENDPOINT_NAME)}
-
-## Response
-
-{generate_return_values_table|zulip.yaml|API_ENDPOINT_NAME}
-
-{generate_response_description(API_ENDPOINT_NAME)}
-
-#### Example response(s)
-
-{generate_code_example|API_ENDPOINT_NAME|fixture}

api_docs/api-keys.md

@@ -1,89 +0,0 @@
-# API keys
-
-An **API key** is how a bot identifies itself to Zulip. For the official
-clients, such as the Python bindings, we recommend [downloading a `zuliprc`
-file](/api/configuring-python-bindings#download-a-zuliprc-file). This file
-contains an API key and other necessary configuration values for using the
-Zulip API with a specific account on a Zulip server.
-
-## Get a bot's API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|your-bots}
-
-1. Click **Active bots**.
-
-1. Find your bot. The bot's API key is under **API KEY**.
-
-{end_tabs}
-
-!!! warn ""
-
-    Anyone with a bot's API key can impersonate the bot, so be careful with it!
-
-## Get your API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|account-and-privacy}
-
-1. Under **API key**, click **Manage your API key**.
-
-1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the instructions from there.
-
-1. Copy your API key.
-
-{end_tabs}
-
-!!! warn ""
-
-    Anyone with your API key can impersonate you, so be doubly careful with it.
-
-
-## Invalidate an API key
-
-To invalidate an existing API key, you have to generate a new key.
-
-### Invalidate a bot's API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|your-bots}
-
-1. Click **Active bots**.
-
-1. Find your bot.
-
-1. Under **API KEY**, click the **refresh** (<i class="fa fa-refresh"></i>) icon
-   to the right of the bot's API key.
-
-{end_tabs}
-
-### Invalidate your API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|account-and-privacy}
-
-1. Under **API key**, click **Manage your API key**.
-
-1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the instructions from there.
-
-1. Click **Generate new API key**
-
-{end_tabs}
-
-## Related articles
-
-* [Configuring the Python bindings](/api/configuring-python-bindings)

api_docs/configuring-python-bindings.md

@@ -1,161 +0,0 @@
-# Configuring the Python bindings
-
-Zulip provides a set of tools that allows interacting with its API more
-easily, called the [Python bindings](https://pypi.python.org/pypi/zulip/).
-One of the most notable use cases for these bindings are bots developed
-using Zulip's [bot framework](/api/writing-bots).
-
-In order to use them, you need to configure them with your identity
-(account, API key, and Zulip server URL). There are a few ways to
-achieve that:
-
-- Using a `zuliprc` file, referenced via the `--config-file` option or
-  the `config_file` option to the `zulip.Client` constructor
-  (recommended for bots).
-- Using a `zuliprc` file in your home directory at `~/.zuliprc`
-  (recommended for your own API key).
-- Using the [environment
-  variables](https://en.wikipedia.org/wiki/Environment_variable)
-  documented below.
-- Using the `--api-key`, `--email`, and `--site` variables as command
-  line parameters.
-- Using the `api_key`, `email`, and `site` parameters to the
-  `zulip.Client` constructor.
-
-## Download a `zuliprc` file
-
-{start_tabs}
-
-{tab|for-a-bot}
-
-{settings_tab|your-bots}
-
-1. Click the **download** (<i class="fa fa-download"></i>) icon on the profile
-   card of the desired bot to download the bot's `zuliprc` file.
-
-!!! warn ""
-
-    Anyone with a bot's API key can impersonate the bot, so be careful with it!
-
-{tab|for-yourself}
-
-{settings_tab|account-and-privacy}
-
-1. Under **API key**, click **Manage your API key**.
-
-1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the
-   instructions from there.
-
-1. Click **Download zuliprc** to download your `zuliprc` file.
-
-1. (optional) If you'd like your credentials to be used by default
-   when using the Zulip API on your computer, move the `zuliprc` file
-   to `~/.zuliprc` in your home directory.
-
-!!! warn ""
-
-    Anyone with your API key can impersonate you, so be doubly careful with it.
-
-{end_tabs}
-
-## Configuration keys and environment variables
-
-`zuliprc` is a configuration file written in the
-[INI file format](https://en.wikipedia.org/wiki/INI_file),
-which contains key-value pairs as shown in the following example:
-
-```
-[api]
-key=<API key from the web interface>
-email=<your email address>
-site=<your Zulip server's URI>
-...
-```
-
-The keys you can use in this file (and their equivalent environment variables)
-can be found in the following table:
-
-<table class="table">
-    <thead>
-        <tr>
-            <th><code>zuliprc</code> key</th>
-            <th>Environment variable</th>
-            <th>Required</th>
-            <th>Description</th>
-        </tr>
-    </thead>
-    <tr>
-        <td><code>key</code></td>
-        <td><code>ZULIP_API_KEY</code></td>
-        <td>Yes</td>
-        <td>
-            <a href="/api/api-keys">API key</a>, which you can get through
-            Zulip's web interface.
-        </td>
-    </tr>
-    <tr>
-        <td><code>email</code></td>
-        <td><code>ZULIP_EMAIL</code></td>
-        <td>Yes</td>
-        <td>
-            The email address of the user who owns the API key mentioned
-            above.
-        </td>
-    </tr>
-    <tr>
-        <td><code>site</code></td>
-        <td><code>ZULIP_SITE</code></td>
-        <td>No</td>
-        <td>
-            URL where your Zulip server is located.
-        </td>
-    </tr>
-    <tr>
-        <td><code>client_cert_key</code></td>
-        <td><code>ZULIP_CERT_KEY</code></td>
-        <td>No</td>
-        <td>
-            Path to the SSL/TLS private key that the binding should use to
-            connect to the server.
-        </td>
-    </tr>
-    <tr>
-        <td><code>client_cert</code></td>
-        <td><code>ZULIP_CERT</code></td>
-        <td>No*</td>
-        <td>
-            The public counterpart of <code>client_cert_key</code>/
-            <code>ZULIP_CERT_KEY</code>. <i>This setting is required if a cert
-            key has been set.</i>
-        </td>
-    </tr>
-    <tr>
-        <td><code>client_bundle</code></td>
-        <td><code>ZULIP_CERT_BUNDLE</code></td>
-        <td>No</td>
-        <td>
-            Path where the server's PEM-encoded certificate is located. CA
-            certificates are also accepted, in case those CA's have issued the
-            server's certificate. Defaults to the built-in CA bundle trusted
-            by Python.
-        </td>
-    </tr>
-    <tr>
-        <td><code>insecure</code></td>
-        <td><code>ZULIP_ALLOW_INSECURE</code></td>
-        <td>No</td>
-        <td>
-            Allows connecting to Zulip servers with an invalid SSL/TLS
-            certificate. Please note that enabling this will make the HTTPS
-            connection insecure. Defaults to <code>false</code>.
-        </td>
-    </tr>
-</table>
-
-## Related articles
-
-* [Installation instructions](/api/installation-instructions)
-* [API keys](/api/api-keys)
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)

api_docs/construct-narrow.md

@@ -1,198 +0,0 @@
-# Construct a narrow
-
-A **narrow** is a set of filters for Zulip messages, that can be based
-on many different factors (like sender, channel, topic, search
-keywords, etc.). Narrows are used in various places in the Zulip
-API (most importantly, in the API for fetching messages).
-
-It is simplest to explain the algorithm for encoding a search as a
-narrow using a single example. Consider the following search query
-(written as it would be entered in the Zulip web app's search box).
-It filters for messages sent to channel `announce`, not sent by
-`iago@zulip.com`, and containing the words `cool` and `sunglasses`:
-
-```
-channel:announce -sender:iago@zulip.com cool sunglasses
-```
-
-This query would be JSON-encoded for use in the Zulip API using JSON
-as a list of simple objects, as follows:
-
-```json
-[
-    {
-        "operator": "channel",
-        "operand": "announce"
-    },
-    {
-        "operator": "sender",
-        "operand": "iago@zulip.com",
-        "negated": true
-    },
-    {
-        "operator": "search",
-        "operand": "cool sunglasses"
-    }
-]
-```
-
-The Zulip help center article on [searching for messages](/help/search-for-messages)
-documents the majority of the search/narrow options supported by the
-Zulip API.
-
-Note that many narrows, including all that lack a `channel` or `channels`
-operator, search the current user's personal message history. See
-[searching shared history](/help/search-for-messages#searching-shared-history)
-for details.
-
-Clients should note that the `is:unread` filter takes advantage of the
-fact that there is a database index for unread messages, which can be an
-important optimization when fetching messages in certain cases (e.g.,
-when [adding the `read` flag to a user's personal
-messages](/api/update-message-flags-for-narrow)).
-
-Note: When the value of `realm_empty_topic_display_name` found in
-the [POST /register](/api/register-queue) response is used as an operand
-for the `"topic"` operator in the narrow, it is interpreted
-as an empty string.
-
-## Changes
-
-* Before Zulip 10.0 (feature level 334), empty string was not a valid
-  topic name for channel messages.
-
-* In Zulip 9.0 (feature level 271), support was added for a new filter
-  operator, `with`, which uses a [message ID](#message-ids) for its
-  operand, and is designed for creating permanent links to topics.
-
-* In Zulip 9.0 (feature level 265), support was added for a new
-  `is:followed` filter, matching messages in topics that the current
-  user is [following](/help/follow-a-topic).
-
-* In Zulip 9.0 (feature level 250), support was added for two filters
-  related to stream messages: `channel` and `channels`. The `channel`
-  operator is an alias for the `stream` operator. The `channels`
-  operator is an alias for the `streams` operator. Both `channel` and
-  `channels` return the same exact results as `stream` and `streams`
-  respectively.
-
-* In Zulip 9.0 (feature level 249), support was added for a new filter,
-  `has:reaction`, which returns messages that have at least one [emoji
-  reaction](/help/emoji-reactions).
-
-* In Zulip 7.0 (feature level 177), support was added for three filters
-  related to direct messages: `is:dm`, `dm` and `dm-including`. The
-  `dm` operator replaced and deprecated the `pm-with` operator. The
-  `is:dm` filter replaced and deprecated the `is:private` filter. The
-  `dm-including` operator replaced and deprecated the `group-pm-with`
-  operator.
-
-    * The `dm-including` and `group-pm-with` operators return slightly
-      different results. For example, `dm-including:1234` returns all
-      direct messages (1-on-1 and group) that include the current user
-      and the user with the unique user ID of `1234`. On the other hand,
-      `group-pm-with:1234` returned only group direct messages that
-      included the current user and the user with the unique user ID of
-      `1234`.
-
-    * Both `dm` and `is:dm` are aliases of `pm-with` and `is:private`
-      respectively, and return the same exact results that the
-      deprecated filters did.
-
-## Narrows that use IDs
-
-### Message IDs
-
-The `id` and `with` operators use message IDs for their operands. The
-message ID operand for these two operators may be encoded as either a
-number or a string.
-
-* `id:12345`: Search for only the message with ID `12345`.
-* `with:12345`: Search for the conversation that contains the message
-  with ID `12345`.
-
-The `id` operator returns the message with the specified ID if it exists,
-and if it can be accessed by the user.
-
-The `with` operator is designed to be used for permanent links to
-topics, which means they should continue to work when the topic is
-[moved](/help/move-content-to-another-topic) or
-[resolved](/help/resolve-a-topic). If the message with the specified
-ID exists, and can be accessed by the user, then it will return
-messages with the `channel`/`topic`/`dm` operators corresponding to
-the current conversation containing that message, replacing any such
-operators included in the original narrow query.
-
-If no such message exists, or the message ID represents a message that
-is inaccessible to the user, this operator will be ignored (rather
-than throwing an error) if the remaining operators uniquely identify a
-conversation (i.e., they contain `channel` and `topic` terms or `dm`
-term). This behavior is intended to provide the best possible
-experience for links to private channels with protected history.
-
-The [help center](/help/search-for-messages#search-by-message-id) also
-documents the `near` operator for searching for messages by ID, but
-this narrow operator has no effect on filtering messages when sent to
-the server. In practice, when the `near` operator is used to search for
-messages, or is part of a URL fragment, the value of its operand should
-instead be used for the value of the `anchor` parameter in endpoints
-that also accept a `narrow` parameter; see
-[GET /messages][anchor-get-messages] and
-[POST /messages/flags/narrow][anchor-post-flags].
-
-**Changes**: Prior to Zulip 8.0 (feature level 194), the message ID
-operand for the `id` operator needed to be encoded as a string.
-
-
-```json
-[
-    {
-        "operator": "id",
-        "operand": 12345
-    }
-]
-```
-
-### Channel and user IDs
-
-There are a few additional narrow/search options (new in Zulip 2.1)
-that use either channel IDs or user IDs that are not documented in the
-help center because they are primarily useful to API clients:
-
-* `channel:1234`: Search messages sent to the channel with ID `1234`.
-* `sender:1234`: Search messages sent by user ID `1234`.
-* `dm:1234`: Search the direct message conversation between
-  you and user ID `1234`.
-* `dm:1234,5678`: Search the direct message conversation between
-  you, user ID `1234`, and user ID `5678`.
-* `dm-including:1234`: Search all direct messages (1-on-1 and group)
-  that include you and user ID `1234`.
-
-!!! tip ""
-
-    A user ID can be found by [viewing a user's profile][view-profile]
-    in the web or desktop apps. A channel ID can be found when [browsing
-    channels][browse-channels] in the web or desktop apps.
-
-The operands for these search options must be encoded either as an
-integer ID or a JSON list of integer IDs. For example, to query
-messages sent by a user 1234 to a direct message thread with yourself,
-user 1234, and user 5678, the correct JSON-encoded query is:
-
-```json
-[
-    {
-        "operator": "dm",
-        "operand": [1234, 5678]
-    },
-    {
-        "operator": "sender",
-        "operand": 1234
-    }
-]
-```
-
-[view-profile]: /help/view-someones-profile
-[browse-channels]: /help/introduction-to-channels#browse-and-subscribe-to-channels
-[anchor-get-messages]: /api/get-messages#parameter-anchor
-[anchor-post-flags]: /api/update-message-flags-for-narrow#parameter-anchor

api_docs/create-scheduled-message.md

@@ -1,49 +0,0 @@
-{generate_api_header(/scheduled_messages:post)}
-
-## Usage examples
-
-{start_tabs}
-
-{generate_code_example(python)|/scheduled_messages:post|example}
-
-{generate_code_example(javascript)|/scheduled_messages:post|example}
-
-{tab|curl}
-
-``` curl
-# Create a scheduled channel message
-curl -X POST {{ api_url }}/v1/scheduled_messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=stream \
-    --data-urlencode to=9 \
-    --data-urlencode topic=Hello \
-    --data-urlencode 'content=Nice to meet everyone!' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-# Create a scheduled direct message
-curl -X POST {{ api_url }}/v1/messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=direct \
-    --data-urlencode 'to=[9, 10]' \
-    --data-urlencode 'content=Can we meet on Monday?' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-```
-
-{end_tabs}
-
-## Parameters
-
-{generate_api_arguments_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_parameter_description(/scheduled_messages:post)}
-
-## Response
-
-{generate_return_values_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_response_description(/scheduled_messages:post)}
-
-#### Example response(s)
-
-{generate_code_example|/scheduled_messages:post|fixture}

api_docs/create-stream.md

@@ -1,6 +0,0 @@
-# Create a channel
-
-You can create a channel using Zulip's REST API by submitting a
-[subscribe](/api/subscribe) request with a channel name that
-doesn't yet exist and passing appropriate parameters to define
-the initial configuration of the new channel.

api_docs/group-setting-values.md

@@ -1,122 +0,0 @@
-# Group-setting values
-
-Settings defining permissions in Zulip are increasingly represented
-using [user groups](/help/user-groups), which offer much more flexible
-configuration than the older [roles](/api/roles-and-permissions) system.
-
-!!! warn ""
-
-    **Note**: Many group-valued settings are configured to require
-    a single system group for their value via
-    `server_supported_permission_settings`, pending web app UI
-    changes to fully support group-setting values.
-
-    **Changes**: Before Zulip 10.0 (feature level 309), only system
-    groups were permitted values for group-setting values in
-    production environments, regardless of the values in
-    `server_supported_permission_settings`.
-
-In the API, these settings are represented using a **group-setting
-value**, which can take two forms:
-
-- An integer user group ID, which can be either a named user group
-  visible in the UI or a [role-based system group](#system-groups).
-- An object with fields `direct_member_ids`, containing a list of
-  integer user IDs, and `direct_subgroup_ids`, containing a list of
-  integer group IDs. The setting's value is the union of the
-  identified collection of users and groups.
-
-Group-setting values in the object form can be thought of as an
-anonymous group. They function very much like a named user group
-object, and remove the naming and UI overhead involved in creating
-a visible user group just to store the value of a single setting.
-
-The server will canonicalize an object with an empty `direct_member_ids`
-list and a `direct_subgroup_ids` list that contains just a single group
-ID to the integer format.
-
-## System groups
-
-The Zulip server maintains a collection of system groups that
-correspond to the users with a given role; this makes it convenient to
-store concepts like "all administrators" in a group-setting
-value. These use a special naming convention and can be recognized by
-the `is_system_group` property on their group object.
-
-The following system groups are maintained by the Zulip server:
-
-- `role:internet`: Everyone on the Internet has this permission; this
-  is used to configure the [public access
-  option](/help/public-access-option).
-- `role:everyone`: All users, including guests.
-- `role:members`: All users, excluding guests.
-- `role:fullmembers`: All [full
-  members](https://zulip.com/api/roles-and-permissions#determining-if-a-user-is-a-full-member)
-  of the organization.
-- `role:moderators`: All users with at least the moderator role.
-- `role:administrators`: All users with at least the administrator
-  role.
-- `role:owners`: All users with the owner role.
-- `role:nobody`: The formal empty group. Used in the API to represent
-  disabling a feature.
-
-Client UI for setting a permission or displaying a group (when
-silently mentioned, for example) is encouraged to display system
-groups using their description, rather than using their `role:}`
-names, which are chosen to be unique and clear in the API.
-
-System groups should generally not be displayed in UI for
-administering an organization's user groups, since they are not
-directly mutable.
-
-## Updating group-setting values
-
-The Zulip API uses a special format for modifying an existing setting
-using a group-setting value.
-
-A **group-setting update** is an object with a `new` field and an
-optional `old` field, each containing a group-setting value. The
-setting's value will be set to the membership expressed by the `new`
-field.
-
-The `old` field expresses the client's understanding of the current
-value of the setting. If the `old` field is present and does not match
-the actual current value of the setting, then the request will fail
-with error code `EXPECTATION_MISMATCH` and no changes will be applied.
-
-When a user edits the setting in a UI, the resulting API request
-should generally always include the `old` field, giving the value
-the list had when the user started editing. This accurately expresses
-the user's intent, and if two users edit the same list around the
-same time, it prevents a situation where the second change
-accidentally reverts the first one without either user noticing.
-
-Omitting `old` is appropriate where the intent really is a new complete
-list rather than an edit, for example in an integration that syncs the
-list from an external source of truth.
-
-## Permitted values
-
-Not every possible group-setting value is a valid configuration for a
-given group-based setting. For example, as a security hardening
-measure, some administrative permissions should never be exercised by
-guest users, and the system group for all users, including guests,
-should not be offered to users as an option for those settings.
-
-Others have restrictions to only permit system groups due to UI
-components not yet having been migrated to support a broader set of
-values. In order to avoid this configuration ending up hardcoded in
-clients, every permission setting using this framework has an entry in
-the `server_supported_permission_settings` section of the [`POST
-/register`](/api/register-queue) response.
-
-Clients that support mutating group-settings values must parse that
-part of the `register` payload in order to compute the set of
-permitted values to offer to the user and avoid server-side errors
-when trying to save a value.
-
-Note specifically that the `allow_everyone_group` field, which
-determines whether the setting can have the value of "all user
-accounts, including guests" also controls whether guests users can
-exercise the permission regardless of their membership in the
-group-setting value.

api_docs/http-headers.md

@@ -1,80 +0,0 @@
-# HTTP headers
-
-This page documents the HTTP headers used by the Zulip API.
-
-Most important is that API clients authenticate to the server using
-HTTP Basic authentication. If you're using the official [Python or
-JavaScript bindings](/api/installation-instructions), this is taken
-care of when you configure said bindings.
-
-Otherwise, see the `curl` example on each endpoint's documentation
-page, which details the request format.
-
-Documented below are additional HTTP headers and header conventions
-generally used by Zulip:
-
-## The `User-Agent` header
-
-Clients are not required to pass a `User-Agent` HTTP header, but we
-highly recommend doing so when writing an integration. It's easy to do
-and it can help save time when debugging issues related to an API
-client.
-
-If provided, the Zulip server will parse the `User-Agent` HTTP header
-in order to identify specific clients and integrations. This
-information is used by the server for logging, [usage
-statistics](/help/analytics), and on rare occasions, for
-backwards-compatibility logic to preserve support for older versions
-of official clients.
-
-Official Zulip clients and integrations use a `User-Agent` that starts
-with something like `ZulipMobile/20.0.103 `, encoding the name of the
-application and it's version.
-
-Zulip's official API bindings have reasonable defaults for
-`User-Agent`. For example, the official Zulip Python bindings have a
-default `User-Agent` starting with `ZulipPython/{version}`, where
-`version` is the version of the library.
-
-You can give your bot/integration its own name by passing the `client`
-parameter when initializing the Python bindings. For example, the
-official Zulip Nagios integration is initialized like this:
-
-``` python
-client = zulip.Client(
-    config_file=opts.config, client=f"ZulipNagios/{VERSION}"
-)
-```
-
-If you are working on an integration that you plan to share outside
-your organization, you can get help picking a good name in
-`#integrations` in the [Zulip development
-community](https://zulip.com/development-community/).
-
-## Rate-limiting response headers
-
-To help clients avoid exceeding rate limits, Zulip sets the following
-HTTP headers in all API responses:
-
-* `X-RateLimit-Remaining`: The number of additional requests of this
-  type that the client can send before exceeding its limit.
-* `X-RateLimit-Limit`: The limit that would be applicable to a client
-  that had not made any recent requests of this type. This is useful
-  for designing a client's burst behavior so as to avoid ever reaching
-  a rate limit.
-* `X-RateLimit-Reset`: The time at which the client will no longer
-  have any rate limits applied to it (and thus could do a burst of
-  `X-RateLimit-Limit` requests).
-
-[Zulip's rate limiting rules are configurable][rate-limiting-rules],
-and can vary by server and over time. The default configuration
-currently limits:
-
-* Every user is limited to 200 total API requests per minute.
-* Separate, much lower limits for authentication/login attempts.
-
-When the Zulip server has configured multiple rate limits that apply
-to a given request, the values returned will be for the strictest
-limit.
-
-[rate-limiting-rules]: https://zulip.readthedocs.io/en/latest/production/security-model.html#rate-limiting

api_docs/include/rest-endpoints.md

@@ -1,144 +0,0 @@
-#### Messages
-
-* [Send a message](/api/send-message)
-* [Upload a file](/api/upload-file)
-* [Edit a message](/api/update-message)
-* [Delete a message](/api/delete-message)
-* [Get messages](/api/get-messages)
-* [Construct a narrow](/api/construct-narrow)
-* [Add an emoji reaction](/api/add-reaction)
-* [Remove an emoji reaction](/api/remove-reaction)
-* [Render a message](/api/render-message)
-* [Fetch a single message](/api/get-message)
-* [Check if messages match a narrow](/api/check-messages-match-narrow)
-* [Get a message's edit history](/api/get-message-history)
-* [Update personal message flags](/api/update-message-flags)
-* [Update personal message flags for narrow](/api/update-message-flags-for-narrow)
-* [Mark all messages as read](/api/mark-all-as-read)
-* [Mark messages in a channel as read](/api/mark-stream-as-read)
-* [Mark messages in a topic as read](/api/mark-topic-as-read)
-* [Get a message's read receipts](/api/get-read-receipts)
-
-#### Scheduled messages
-
-* [Get scheduled messages](/api/get-scheduled-messages)
-* [Create a scheduled message](/api/create-scheduled-message)
-* [Edit a scheduled message](/api/update-scheduled-message)
-* [Delete a scheduled message](/api/delete-scheduled-message)
-
-#### Drafts
-
-* [Get drafts](/api/get-drafts)
-* [Create drafts](/api/create-drafts)
-* [Edit a draft](/api/edit-draft)
-* [Delete a draft](/api/delete-draft)
-* [Get all saved snippets](/api/get-saved-snippets)
-* [Create a saved snippet](/api/create-saved-snippet)
-* [Delete a saved snippet](/api/delete-saved-snippet)
-
-#### Channels
-
-* [Get subscribed channels](/api/get-subscriptions)
-* [Subscribe to a channel](/api/subscribe)
-* [Unsubscribe from a channel](/api/unsubscribe)
-* [Get subscription status](/api/get-subscription-status)
-* [Get channel subscribers](/api/get-subscribers)
-* [Update subscription settings](/api/update-subscription-settings)
-* [Get all channels](/api/get-streams)
-* [Get a channel by ID](/api/get-stream-by-id)
-* [Get channel ID](/api/get-stream-id)
-* [Create a channel](/api/create-stream)
-* [Update a channel](/api/update-stream)
-* [Archive a channel](/api/archive-stream)
-* [Get channel's email address](/api/get-stream-email-address)
-* [Get topics in a channel](/api/get-stream-topics)
-* [Topic muting](/api/mute-topic)
-* [Update personal preferences for a topic](/api/update-user-topic)
-* [Delete a topic](/api/delete-topic)
-* [Add a default channel](/api/add-default-stream)
-* [Remove a default channel](/api/remove-default-stream)
-
-#### Users
-
-* [Get a user](/api/get-user)
-* [Get a user by email](/api/get-user-by-email)
-* [Get own user](/api/get-own-user)
-* [Get all users](/api/get-users)
-* [Create a user](/api/create-user)
-* [Update a user](/api/update-user)
-* [Update a user by email](/api/update-user-by-email)
-* [Deactivate a user](/api/deactivate-user)
-* [Deactivate own user](/api/deactivate-own-user)
-* [Reactivate a user](/api/reactivate-user)
-* [Get a user's status](/api/get-user-status)
-* [Update your status](/api/update-status)
-* [Set "typing" status](/api/set-typing-status)
-* [Set "typing" status for message editing](/api/set-typing-status-for-message-edit)
-* [Get a user's presence](/api/get-user-presence)
-* [Get presence of all users](/api/get-presence)
-* [Update your presence](/api/update-presence)
-* [Get attachments](/api/get-attachments)
-* [Delete an attachment](/api/remove-attachment)
-* [Update settings](/api/update-settings)
-* [Get user groups](/api/get-user-groups)
-* [Create a user group](/api/create-user-group)
-* [Update a user group](/api/update-user-group)
-* [Deactivate a user group](/api/deactivate-user-group)
-* [Update user group members](/api/update-user-group-members)
-* [Update subgroups of a user group](/api/update-user-group-subgroups)
-* [Get user group membership status](/api/get-is-user-group-member)
-* [Get user group members](/api/get-user-group-members)
-* [Get subgroups of a user group](/api/get-user-group-subgroups)
-* [Mute a user](/api/mute-user)
-* [Unmute a user](/api/unmute-user)
-* [Get all alert words](/api/get-alert-words)
-* [Add alert words](/api/add-alert-words)
-* [Remove alert words](/api/remove-alert-words)
-
-#### Invitations
-
-* [Get all invitations](/api/get-invites)
-* [Send invitations](/api/send-invites)
-* [Create a reusable invitation link](/api/create-invite-link)
-* [Resend an email invitation](/api/resend-email-invite)
-* [Revoke an email invitation](/api/revoke-email-invite)
-* [Revoke a reusable invitation link](/api/revoke-invite-link)
-
-#### Server & organizations
-
-* [Get server settings](/api/get-server-settings)
-* [Get linkifiers](/api/get-linkifiers)
-* [Add a linkifier](/api/add-linkifier)
-* [Update a linkifier](/api/update-linkifier)
-* [Remove a linkifier](/api/remove-linkifier)
-* [Reorder linkifiers](/api/reorder-linkifiers)
-* [Add a code playground](/api/add-code-playground)
-* [Remove a code playground](/api/remove-code-playground)
-* [Get all custom emoji](/api/get-custom-emoji)
-* [Upload custom emoji](/api/upload-custom-emoji)
-* [Deactivate custom emoji](/api/deactivate-custom-emoji)
-* [Get all custom profile fields](/api/get-custom-profile-fields)
-* [Reorder custom profile fields](/api/reorder-custom-profile-fields)
-* [Create a custom profile field](/api/create-custom-profile-field)
-* [Update realm-level defaults of user settings](/api/update-realm-user-settings-defaults)
-* [Get all data exports](/api/get-realm-exports)
-* [Create a data export](/api/export-realm)
-* [Get data export consent state](/api/get-realm-export-consents)
-
-#### Real-time events
-
-* [Real time events API](/api/real-time-events)
-* [Register an event queue](/api/register-queue)
-* [Get events from an event queue](/api/get-events)
-* [Delete an event queue](/api/delete-queue)
-
-#### Specialty endpoints
-
-* [Fetch an API key (production)](/api/fetch-api-key)
-* [Fetch an API key (development only)](/api/dev-fetch-api-key)
-* [Send a test notification to mobile device(s)](/api/test-notify)
-* [Add an APNs device token](/api/add-apns-token)
-* [Remove an APNs device token](/api/remove-apns-token)
-* [Add an FCM registration token](/api/add-fcm-token)
-* [Remove an FCM registration token](/api/remove-fcm-token)
-* [Create BigBlueButton video call](/api/create-big-blue-button-video-call)

api_docs/incoming-webhooks-overview.md

@@ -1,223 +0,0 @@
-# Incoming webhook integrations
-
-An incoming webhook allows a third-party service to push data to Zulip when
-something happens. There are several ways to set up an incoming webhook in
-Zulip:
-
-* Use our [REST API](/api/rest) endpoint for [sending
-  messages](/api/send-message).  This works great for internal tools
-  or cases where the third-party tool wants to control the formatting
-  of the messages in Zulip.
-* Use one of our supported [integration
-  frameworks](/integrations/meta-integration), such as the
-  [Slack-compatible incoming webhook](/integrations/doc/slack_incoming),
-  [Zapier integration](/integrations/doc/zapier), or
-  [IFTTT integration](/integrations/doc/ifttt).
-* Implementing an incoming webhook integration (detailed on this page),
-  where all the logic for formatting the Zulip messages lives in the
-  Zulip server.  This is how most of [Zulip's official
-  integrations](/integrations/) work, because they enable Zulip to
-  support third-party services that just have an "outgoing webhook"
-  feature (without the third party needing to do any work specific to
-  Zulip).
-
-In an incoming webhook integration, the third-party service's
-"outgoing webhook" feature sends an `HTTP POST` to a special URL when
-it has something for you, and then the Zulip "incoming webhook"
-integration handles that incoming data to format and send a message in
-Zulip.
-
-New official Zulip webhook integrations can take just a few hours to
-write, including tests and documentation, if you use the right
-process.
-
-## Quick guide
-
-* Set up the
-  [Zulip development environment](https://zulip.readthedocs.io/en/latest/development/overview.html).
-
-* Use [Zulip's JSON integration](/integrations/doc/json),
-  <https://webhook.site/>, or a similar site to capture an example
-  webhook payload from the third-party service. Create a
-  `zerver/webhooks/<mywebhook>/fixtures/` directory, and add the
-  captured JSON payload as a test fixture.
-
-* Create an `Integration` object, and add it to the `WEBHOOK_INTEGRATIONS`
-  list in `zerver/lib/integrations.py`. Search for `WebhookIntegration` in that
-  file to find an existing one to copy.
-
-* Write a draft webhook handler in `zerver/webhooks/<mywebhook>/view.py`. There
-  are a lot of examples in the `zerver/webhooks/` directory that you can copy.
-  We recommend templating from a short one, like `zendesk`.
-
-* Write a test for your fixture in `zerver/webhooks/<mywebhook>/tests.py`.
-  Run the test for your integration like this:
-
-    ```
-    tools/test-backend zerver/webhooks/<mywebhook>/
-    ```
-
-    Iterate on debugging the test and webhooks handler until it all
-    works.
-
-* Capture payloads for the other common types of `POST`s the third-party
-  service will make, and add tests for them; usually this part of the
-  process is pretty fast.
-
-* Document the integration in `zerver/webhooks/<mywebhook>/doc.md`(required for
-  getting it merged into Zulip). You can use existing documentation, like
-  [this one](https://raw.githubusercontent.com/zulip/zulip/main/zerver/webhooks/github/doc.md),
-  as a template. This should not take more than 15 minutes, even if you don't speak English
-  as a first language (we'll clean up the text before merging).
-
-## Hello world walkthrough
-
-Check out the [detailed walkthrough](incoming-webhooks-walkthrough) for step-by-step
-instructions.
-
-## Checklist
-
-### Files that need to be created
-
-Select a name for your incoming webhook and use it consistently. The examples
-below are for a webhook named `MyWebHook`.
-
-* `zerver/webhooks/mywebhook/__init__.py`: Empty file that is an obligatory
-   part of every python package.  Remember to `git add` it.
-* `zerver/webhooks/mywebhook/view.py`: The main webhook integration function,
-  called `api_mywebhook_webhook`, along with any necessary helper functions.
-* `zerver/webhooks/mywebhook/fixtures/message_type.json`: Sample JSON payload data
-  used by tests. Add one fixture file per type of message supported by your
-  integration.
-* `zerver/webhooks/mywebhook/tests.py`: Tests for your webhook.
-* `zerver/webhooks/mywebhook/doc.md`: End-user documentation explaining
-  how to add the integration.
-* `static/images/integrations/logos/mywebhook.svg`: A square logo for the
-  platform/server/product you are integrating. Used on the documentation
-  pages as well as the sender's avatar for messages sent by the integration.
-* `static/images/integrations/mywebhook/001.png`: A screenshot of a message
-  sent by the integration, used on the documentation page. This can be
-  generated by running `tools/screenshots/generate-integration-docs-screenshot --integration mywebhook`.
-* `static/images/integrations/bot_avatars/mywebhook.png`: A square logo for the
-  platform/server/product you are integrating which is used to create the avatar
-  for generating screenshots with. This can be generated automatically from
-  `static/images/integrations/logos/mywebhook.svg` by running
-  `tools/setup/generate_integration_bots_avatars.py`.
-
-### Files that need to be updated
-
-* `zerver/lib/integrations.py`: Add your integration to
-  `WEBHOOK_INTEGRATIONS`. This will automatically register a
-  URL for the incoming webhook of the form `api/v1/external/mywebhook` and
-  associate it with the function called `api_mywebhook_webhook` in
-  `zerver/webhooks/mywebhook/view.py`. Also add your integration to
-  `DOC_SCREENSHOT_CONFIG`. This will allow you to automatically generate
-  a screenshot for the documentation by running
-  `tools/screenshots/generate-integration-docs-screenshot --integration mywebhook`.
-
-## Common Helpers
-
-* If your integration will receive a test webhook payload, you can use
-  `get_setup_webhook_message` to create our standard message for test payloads.
-  You can import this from `zerver/lib/webhooks/common.py`, and it will generate
-  a message like this: "GitHub webhook is successfully configured! 🎉"
-
-## General advice
-
-* Consider using our Zulip markup to make the output from your
-  integration especially attractive or useful (e.g., emoji, Markdown
-  emphasis, or @-mentions).
-
-* Use topics effectively to ensure sequential messages about the same
-  thing are threaded together; this makes for much better consumption
-  by users.  E.g., for a bug tracker integration, put the bug number in
-  the topic for all messages; for an integration like Nagios, put the
-  service in the topic.
-
-* Integrations that don't match a team's workflow can often be
-  uselessly spammy.  Give careful thought to providing options for
-  triggering Zulip messages only for certain message types, certain
-  projects, or sending different messages to different channels/topics,
-  to make it easy for teams to configure the integration to support
-  their workflow.
-
-* Consistently capitalize the name of the integration in the
-  documentation and the Client name the way the vendor does.  It's OK
-  to use all-lower-case in the implementation.
-
-* Sometimes it can be helpful to contact the vendor if it appears they
-  don't have an API or webhook we can use; sometimes the right API
-  is just not properly documented.
-
-* A helpful tool for testing your integration is
-  [UltraHook](http://www.ultrahook.com/), which allows you to receive webhook
-  calls via your local Zulip development environment. This enables you to do end-to-end
-  testing with live data from the service you're integrating and can help you
-  spot why something isn't working or if the service is using custom HTTP
-  headers.
-
-## URL specification
-
-The base URL for an incoming webhook integration bot, where
-`INTEGRATION_NAME` is the name of the specific webhook integration and
-`API_KEY` is the API key of the bot created by the user for the
-integration, is:
-
-```
-{{ api_url }}/v1/external/INTEGRATION_NAME?api_key=API_KEY
-```
-
-The list of existing webhook integrations can be found by browsing the
-[Integrations documentation](/integrations/) or in
-`zerver/lib/integrations.py` at `WEBHOOK_INTEGRATIONS`.
-
-Parameters accepted in the URL include:
-
-### api_key *(required)*
-
-The API key of the bot created by the user for the integration. To get a
-bot's API key, see the [API keys](/api/api-keys) documentation.
-
-### stream
-
-The channel for the integration to send notifications to. Can be either
-the channel ID or the [URL-encoded][url-encoder] channel name. By default
-the integration will send direct messages to the bot's owner.
-
-!!! tip ""
-
-    A channel ID can be found when [browsing channels][browse-channels]
-    in the web or desktop apps.
-
-### topic
-
-The topic in the specified channel for the integration to send
-notifications to. The topic should also be [URL-encoded][url-encoder].
-By default the integration will have a topic configured for channel
-messages.
-
-### only_events, exclude_events
-
-Some incoming webhook integrations support these parameters to filter
-which events will trigger a notification. You can append either
-`&only_events=["event_a","event_b"]` or
-`&exclude_events=["event_a","event_b"]` (or both, with different events)
-to the URL, with an arbitrary number of supported events.
-
-You can use UNIX-style wildcards like `*` to include multiple events.
-For example, `test*` matches every event that starts with `test`.
-
-!!! tip ""
-
-    For a list of supported events, see a specific [integration's
-    documentation](/integrations) page.
-
-[browse-channels]: /help/introduction-to-channels#browse-and-subscribe-to-channels
-[add-bot]: /help/add-a-bot-or-integration
-[url-encoder]: https://www.urlencoder.org/
-
-## Related articles
-
-* [Integrations overview](/api/integrations-overview)
-* [Incoming webhook walkthrough](/api/incoming-webhooks-walkthrough)
-* [Non-webhook integrations](/api/non-webhook-integrations)

api_docs/incoming-webhooks-walkthrough.md

@@ -1,661 +0,0 @@
-# Incoming webhook walkthrough
-
-Below, we explain each part of a simple incoming webhook integration,
-called **Hello World**.  This integration sends a "hello" message to the `test`
-channel and includes a link to the Wikipedia article of the day, which
-it formats from json data it receives in the http request.
-
-Use this walkthrough to learn how to write your first webhook
-integration.
-
-## Step 0: Create fixtures
-
-The first step in creating an incoming webhook is to examine the data that the
-service you want to integrate will be sending to Zulip.
-
-* Use [Zulip's JSON integration](/integrations/doc/json),
-<https://webhook.site/>, or a similar tool to capture webhook
-payload(s) from the service you are integrating. Examining this data
-allows you to do two things:
-
-1. Determine how you will need to structure your webhook code, including what
-   message types your integration should support and how.
-2. Create fixtures for your webhook tests.
-
-A test fixture is a small file containing test data, one for each test.
-Fixtures enable the testing of webhook integration code without the need to
-actually contact the service being integrated.
-
-Because `Hello World` is a very simple integration that does one
-thing, it requires only one fixture,
-`zerver/webhooks/helloworld/fixtures/hello.json`:
-
-```json
-{
-  "featured_title":"Marilyn Monroe",
-  "featured_url":"https://en.wikipedia.org/wiki/Marilyn_Monroe",
-}
-```
-
-When writing your own incoming webhook integration, you'll want to write a test function
-for each distinct message condition your integration supports. You'll also need a
-corresponding fixture for each of these tests. Depending on the type of data
-the 3rd party service sends, your fixture may contain JSON, URL encoded text, or
-some other kind of data. See [Step 5: Create automated tests](#step-5-create-automated-tests) or
-[Testing](https://zulip.readthedocs.io/en/latest/testing/testing.html) for further details.
-
-### HTTP Headers
-
-Some third-party webhook APIs, such as GitHub's, don't encode all the
-information about an event in the JSON request body.  Instead, they
-put key details like the event type in a separate HTTP header
-(generally this is clear in their API documentation).  In order to
-test Zulip's handling of that integration, you will need to record
-which HTTP headers are used with each fixture you capture.
-
-Since this is integration-dependent, Zulip offers a simple API for
-doing this, which is probably best explained by looking at the example
-for GitHub: `zerver/webhooks/github/view.py`; basically, as part of
-writing your integration, you'll write a special function in your
-view.py file that maps the filename of the fixture to the set of HTTP
-headers to use. This function must be named "fixture_to_headers". Most
-integrations will use the same strategy as the GitHub integration:
-encoding the third party variable header data (usually just an event
-type) in the fixture filename, in such a case, you won't need to
-explicitly write the logic for such a special function again,
-instead you can just use the same helper method that the GitHub
-integration uses.
-
-## Step 1: Initialize your webhook python package
-
-In the `zerver/webhooks/` directory, create new subdirectory that will
-contain all of the corresponding code. In our example, it will be
-`helloworld`. The new directory will be a python package, so you have
-to create an empty `__init__.py` file in that directory via, for
-example, `touch zerver/webhooks/helloworld/__init__.py`.
-
-## Step 2: Create main webhook code
-
-The majority of the code for your new integration will be in a single
-python file, `zerver/webhooks/mywebhook/view.py`.
-
-The Hello World integration is in `zerver/webhooks/helloworld/view.py`:
-
-```python
-from django.http import HttpRequest, HttpResponse
-
-from zerver.decorator import webhook_view
-from zerver.lib.response import json_success
-from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
-from zerver.lib.validator import WildValue, check_string
-from zerver.lib.webhooks.common import check_send_webhook_message
-from zerver.models import UserProfile
-
-
-@webhook_view("HelloWorld")
-@typed_endpoint
-def api_helloworld_webhook(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    *,
-    payload: JsonBodyPayload[WildValue],
-) -> HttpResponse:
-    # construct the body of the message
-    body = "Hello! I am happy to be here! :smile:"
-
-    # try to add the Wikipedia article of the day
-    body_template = (
-        "\nThe Wikipedia featured article for today is **[{featured_title}]({featured_url})**"
-    )
-    body += body_template.format(
-        featured_title=payload["featured_title"].tame(check_string),
-        featured_url=payload["featured_url"].tame(check_string),
-    )
-
-    topic = "Hello World"
-
-    # send the message
-    check_send_webhook_message(request, user_profile, topic, body)
-
-    return json_success(request)
-```
-
-The above code imports the required functions and defines the main webhook
-function `api_helloworld_webhook`, decorating it with `webhook_view` and
-`typed_endpoint`. The `typed_endpoint` decorator allows you to
-access request variables with `JsonBodyPayload()`. You can find more about `JsonBodyPayload` and request variables in [Writing views](
-https://zulip.readthedocs.io/en/latest/tutorials/writing-views.html#request-variables).
-
-You must pass the name of your integration to the
-`webhook_view` decorator; that name will be used to
-describe your integration in Zulip's analytics (e.g., the `/stats`
-page). Here we have used `HelloWorld`. To be consistent with other
-integrations, use the name of the product you are integrating in camel
-case, spelled as the product spells its own name (except always first
-letter upper-case).
-
-The `webhook_view` decorator indicates that the 3rd party service will
-send the authorization as an API key in the query parameters. If your service uses
-HTTP basic authentication, you would instead use the `authenticated_rest_api_view`
-decorator.
-
-You should name your webhook function as such
-`api_webhookname_webhook` where `webhookname` is the name of your
-integration and is always lower-case.
-
-At minimum, the webhook function must accept `request` (Django
-[HttpRequest](https://docs.djangoproject.com/en/5.0/ref/request-response/#django.http.HttpRequest)
-object), and `user_profile` (Zulip's user object). You may also want to
-define additional parameters using the `typed_endpoint` decorator.
-
-In the example above, we have defined `payload` which is populated
-from the body of the http request, `stream` with a default of `test`
-(available by default in the Zulip development environment), and
-`topic` with a default of `Hello World`. If your webhook uses a custom channel,
-it must exist before a message can be created in it. (See
-[Step 4: Create automated tests](#step-5-create-automated-tests) for how to handle this in tests.)
-
-The line that begins `# type` is a mypy type annotation. See [this
-page](https://zulip.readthedocs.io/en/latest/testing/mypy.html) for details about
-how to properly annotate your webhook functions.
-
-In the body of the function we define the body of the message as `Hello! I am
-happy to be here! :smile:`. The `:smile:` indicates an emoji. Then we append a
-link to the Wikipedia article of the day as provided by the json payload.
-
-* Sometimes, it might occur that a json payload does not contain all required keys your
-  integration checks for. In such a case, any `KeyError` thrown is handled by the server
-  backend and will create an appropriate response.
-
-Then we send a message with `check_send_webhook_message`, which will
-validate the message and do the following:
-
-* Send a public (channel) message if the `stream` query parameter is
-  specified in the webhook URL.
-* If the `stream` query parameter isn't specified, it will send a direct
-  message to the owner of the webhook bot.
-
-Finally, we return a 200 http status with a JSON format success message via
-`json_success(request)`.
-
-## Step 3: Create an API endpoint for the webhook
-
-In order for an incoming webhook to be externally available, it must be mapped
-to a URL. This is done in `zerver/lib/integrations.py`.
-
-Look for the lines beginning with:
-
-```python
-WEBHOOK_INTEGRATIONS: List[WebhookIntegration] = [
-```
-
-And you'll find the entry for Hello World:
-
-```python
-  WebhookIntegration("helloworld", ["misc"], display_name="Hello World"),
-```
-
-This tells the Zulip API to call the `api_helloworld_webhook` function in
-`zerver/webhooks/helloworld/view.py` when it receives a request at
-`/api/v1/external/helloworld`.
-
-This line also tells Zulip to generate an entry for Hello World on the Zulip
-integrations page using `static/images/integrations/logos/helloworld.svg` as its
-icon. The second positional argument defines a list of categories for the
-integration.
-
-At this point, if you're following along and/or writing your own Hello World
-webhook, you have written enough code to test your integration. There are three
-tools which you can use to test your webhook - 2 command line tools and a GUI.
-
-### Webhooks requiring custom configuration
-
-In rare cases, it's necessary for an incoming webhook to require
-additional user configuration beyond what is specified in the post
-URL.  The typical use case for this is APIs like the Stripe API that
-require clients to do a callback to get details beyond an opaque
-object ID that one would want to include in a Zulip notification.
-
-These configuration options are declared as follows:
-
-```python
-    WebhookIntegration('helloworld', ['misc'], display_name='Hello World',
-                       config_options=[('HelloWorld API key', 'hw_api_key', check_string)])
-```
-
-`config_options` is a list describing the parameters the user should
-configure:
-    1. A user-facing string describing the field to display to users.
-    2. The field name you'll use to access this from your `view.py` function.
-    3. A Validator, used to verify the input is valid.
-
-Common validators are available in `zerver/lib/validators.py`.
-
-## Step 4: Manually testing the webhook
-
-For either one of the command line tools, first, you'll need to get an
-API key from the **Bots** section of your Zulip user's **Personal
-settings**. To test the webhook, you'll need to [create a
-bot](https://zulip.com/help/add-a-bot-or-integration) with the
-**Incoming webhook** type. Replace `<api_key>` with your bot's API key
-in the examples presented below! This is how Zulip knows that the
-request was made by an authorized user.
-
-### Curl
-
-Using curl:
-```bash
-curl -X POST -H "Content-Type: application/json" -d '{ "featured_title":"Marilyn Monroe", "featured_url":"https://en.wikipedia.org/wiki/Marilyn_Monroe" }' http://localhost:9991/api/v1/external/helloworld\?api_key\=<api_key>
-```
-
-After running the above command, you should see something similar to:
-
-```json
-{"msg":"","result":"success"}
-```
-
-### Management command: send_webhook_fixture_message
-
-Using `manage.py` from within the Zulip development environment:
-
-```console
-(zulip-server) vagrant@vagrant:/srv/zulip$
-./manage.py send_webhook_fixture_message \
-    --fixture=zerver/webhooks/helloworld/fixtures/hello.json \
-    '--url=http://localhost:9991/api/v1/external/helloworld?api_key=<api_key>'
-```
-
-After running the above command, you should see something similar to:
-
-```
-2016-07-07 15:06:59,187 INFO     127.0.0.1       POST    200 143ms (mem: 6ms/13) (md: 43ms/1) (db: 20ms/9q) (+start: 147ms) /api/v1/external/helloworld (helloworld-bot@zulip.com via ZulipHelloWorldWebhook)
-```
-
-Some webhooks require custom HTTP headers, which can be passed using
-`./manage.py send_webhook_fixture_message --custom-headers`.  For
-example:
-
-    --custom-headers='{"X-Custom-Header": "value"}'
-
-The format is a JSON dictionary, so make sure that the header names do
-not contain any spaces in them and that you use the precise quoting
-approach shown above.
-
-For more information about `manage.py` command-line tools in Zulip, see
-the [management commands][management-commands] documentation.
-
-[management-commands]: https://zulip.readthedocs.io/en/latest/production/management-commands.html
-
-### Integrations Dev Panel
-This is the GUI tool.
-
-{start_tabs}
-
-1. Run `./tools/run-dev` then go to http://localhost:9991/devtools/integrations/.
-
-1. Set the following mandatory fields:
-**Bot** - Any incoming webhook bot.
-**Integration** - One of the integrations.
-**Fixture** - Though not mandatory, it's recommended that you select one and then tweak it if necessary.
-The remaining fields are optional, and the URL will automatically be generated.
-
-1. Click **Send**!
-
-{end_tabs}
-
-By opening Zulip in one tab and then this tool in another, you can quickly tweak
-your code and send sample messages for many different test fixtures.
-
-Note: Custom HTTP Headers must be entered as a JSON dictionary, if you want to use any in the first place that is.
-Feel free to use 4-spaces as tabs for indentation if you'd like!
-
-Your sample notification may look like:
-
-<img class="screenshot" src="/static/images/api/helloworld-webhook.png" alt="screenshot" />
-
-
-
-## Step 5: Create automated tests
-
-Every webhook integration should have a corresponding test file:
-`zerver/webhooks/mywebhook/tests.py`.
-
-The Hello World integration's tests are in `zerver/webhooks/helloworld/tests.py`
-
-You should name the class `<WebhookName>HookTests` and have it inherit from
-the base class `WebhookTestCase`. For our HelloWorld webhook, we name the test
-class `HelloWorldHookTests`:
-
-```python
-class HelloWorldHookTests(WebhookTestCase):
-    CHANNEL_NAME = "test"
-    URL_TEMPLATE = "/api/v1/external/helloworld?&api_key={api_key}&stream={stream}"
-    DIRECT_MESSAGE_URL_TEMPLATE = "/api/v1/external/helloworld?&api_key={api_key}"
-    WEBHOOK_DIR_NAME = "helloworld"
-
-    # Note: Include a test function per each distinct message condition your integration supports
-    def test_hello_message(self) -> None:
-        expected_topic = "Hello World"
-        expected_message = "Hello! I am happy to be here! :smile:\nThe Wikipedia featured article for today is **[Marilyn Monroe](https://en.wikipedia.org/wiki/Marilyn_Monroe)**"
-
-        # use fixture named helloworld_hello
-        self.check_webhook(
-            "hello",
-            expected_topic,
-            expected_message,
-            content_type="application/x-www-form-urlencoded",
-        )
-```
-
-In the above example, `CHANNEL_NAME`, `URL_TEMPLATE`, and `WEBHOOK_DIR_NAME` refer
-to class attributes from the base class, `WebhookTestCase`. These are needed by
-the helper function `check_webhook` to determine how to execute
-your test. `CHANNEL_NAME` should be set to your default channel. If it doesn't exist,
-`check_webhook` will create it while executing your test.
-
-If your test expects a channel name from a test fixture, the value in the fixture
-and the value you set for `CHANNEL_NAME` must match. The test helpers use `CHANNEL_NAME`
-to create the destination channel, and then create the message to send using the
-value from the fixture. If these don't match, the test will fail.
-
-`URL_TEMPLATE` defines how the test runner will call your incoming webhook, in the same way
- you would provide a webhook URL to the 3rd party service. `api_key={api_key}` says
-that an API key is expected.
-
-When writing tests for your webhook, you'll want to include one test function
-(and corresponding fixture) per each distinct message condition that your
-integration supports.
-
-If, for example, we added support for sending a goodbye message to our `Hello
-World` webhook, we would add another test function to `HelloWorldHookTests`
-class called something like `test_goodbye_message`:
-
-```python
-    def test_goodbye_message(self) -> None:
-        expected_topic = "Hello World"
-        expected_message = "Hello! I am happy to be here! :smile:\nThe Wikipedia featured article for today is **[Goodbye](https://en.wikipedia.org/wiki/Goodbye)**"
-
-        # use fixture named helloworld_goodbye
-        self.check_webhook(
-            "goodbye",
-            expected_topic,
-            expected_message,
-            content_type="application/x-www-form-urlencoded",
-        )
-```
-
-As well as a new fixture `goodbye.json` in
-`zerver/webhooks/helloworld/fixtures/`:
-
-```json
-{
-  "featured_title":"Goodbye",
-  "featured_url":"https://en.wikipedia.org/wiki/Goodbye",
-}
-```
-
-Also consider if your integration should have negative tests, a test where the
-data from the test fixture should result in an error. For details see
-[Negative tests](#negative-tests), below.
-
-Once you have written some tests, you can run just these new tests from within
-the Zulip development environment with this command:
-
-```console
-(zulip-server) vagrant@vagrant:/srv/zulip$
-./tools/test-backend zerver/webhooks/helloworld
-```
-
-(Note: You must run the tests from the top level of your development directory.
-The standard location in a Vagrant environment is `/srv/zulip`. If you are not
-using Vagrant, use the directory where you have your development environment.)
-
-You will see some script output and if all the tests have passed, you will see:
-
-```console
-Running zerver.webhooks.helloworld.tests.HelloWorldHookTests.test_goodbye_message
-Running zerver.webhooks.helloworld.tests.HelloWorldHookTests.test_hello_message
-DONE!
-```
-
-## Step 6: Create documentation
-
-Next, we add end-user documentation for our integration.  You
-can see the existing examples at <https://zulip.com/integrations>
-or by accessing `/integrations` in your Zulip development environment.
-
-There are two parts to the end-user documentation on this page.
-
-The first is the lozenge in the grid of integrations, showing your
-integration logo and name, which links to the full documentation.
-This is generated automatically once you've registered the integration
-in `WEBHOOK_INTEGRATIONS` in `zerver/lib/integrations.py`, and supports
-some customization via options to the `WebhookIntegration` class.
-
-Second, you need to write the actual documentation content in
-`zerver/webhooks/mywebhook/doc.md`.
-
-```md
-Learn how Zulip integrations work with this simple Hello World example!
-
-1.  The Hello World webhook will use the `test` channel, which is created
-    by default in the Zulip development environment. If you are running
-    Zulip in production, you should make sure that this channel exists.
-
-1. {!create-an-incoming-webhook.md!}
-
-1. {!generate-webhook-url-basic.md!}
-
-1.  To trigger a notification using this example webhook, you can use
-    `send_webhook_fixture_message` from a [Zulip development
-    environment](https://zulip.readthedocs.io/en/latest/development/overview.html):
-
-    ```
-        (zulip-server) vagrant@vagrant:/srv/zulip$
-        ./manage.py send_webhook_fixture_message \
-        > --fixture=zerver/tests/fixtures/helloworld/hello.json \
-        > '--url=http://localhost:9991/api/v1/external/helloworld?api_key=abcdefgh&stream=stream%20name;'
-    ```
-
-    Or, use curl:
-
-    ```
-    curl -X POST -H "Content-Type: application/json" -d '{ "featured_title":"Marilyn Monroe", "featured_url":"https://en.wikipedia.org/wiki/Marilyn_Monroe" }' http://localhost:9991/api/v1/external/helloworld\?api_key=abcdefgh&stream=stream%20name;
-    ```
-
-{!congrats.md!}
-
-![Hello World integration](/static/images/integrations/helloworld/001.png)
-
-```
-
-`{!create-an-incoming-webhook.md!}` and `{!congrats.md!}` are examples of
-a Markdown macro. Zulip has a macro-based Markdown/Jinja2 framework that
-includes macros for common instructions in Zulip's webhooks/integrations
-documentation.
-
-See
-[our guide on documenting an integration][integration-docs-guide]
-for further details, including how to easily create the message
-screenshot. Mostly you should plan on templating off an existing guide, like
-[this one](https://raw.githubusercontent.com/zulip/zulip/main/zerver/webhooks/github/doc.md).
-
-[integration-docs-guide]: https://zulip.readthedocs.io/en/latest/documentation/integrations.html
-
-## Step 7: Preparing a pull request to zulip/zulip
-
-When you have finished your webhook integration, follow these guidelines before
-pushing the code to your fork and submitting a pull request to zulip/zulip:
-
-- Run tests including linters and ensure you have addressed any issues they
-  report. See [Testing](https://zulip.readthedocs.io/en/latest/testing/testing.html)
-  and [Linters](https://zulip.readthedocs.io/en/latest/testing/linters.html) for details.
-- Read through [Code styles and conventions](
-  https://zulip.readthedocs.io/en/latest/contributing/code-style.html) and take a look
-  through your code to double-check that you've followed Zulip's guidelines.
-- Take a look at your Git history to ensure your commits have been clear and
-  logical (see [Commit discipline](
-  https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html) for tips). If not,
-  consider revising them with `git rebase --interactive`. For most incoming webhooks,
-  you'll want to squash your changes into a single commit and include a good,
-  clear commit message.
-
-If you would like feedback on your integration as you go, feel free to post a
-message on the [public Zulip instance](https://chat.zulip.org/#narrow/channel/integrations).
-You can also create a [draft pull request](
-https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) while you
-are still working on your integration. See the
-[Git guide](https://zulip.readthedocs.io/en/latest/git/pull-requests.html#create-a-pull-request)
-for more on Zulip's pull request process.
-
-## Advanced topics
-
-More complex implementation or testing needs may require additional code, beyond
-what the standard helper functions provide. This section discusses some of
-these situations.
-
-### Negative tests
-
-A negative test is one that should result in an error, such as incorrect data.
-The helper functions may interpret this as a test failure, when it should instead
-be a successful test of an error condition. To correctly test these cases, you
-must explicitly code your test's execution (using other helpers, as needed)
-rather than call the usual helper function.
-
-Here is an example from the WordPress integration:
-
-```python
-def test_unknown_action_no_data(self) -> None:
-    # Mimic check_webhook() to manually execute a negative test.
-    # Otherwise its call to send_webhook_payload() would assert on the non-success
-    # we are testing. The value of result is the error message the webhook should
-    # return if no params are sent. The fixture for this test is an empty file.
-
-    # subscribe to the target channel
-    self.subscribe(self.test_user, self.CHANNEL_NAME)
-
-    # post to the webhook url
-    post_params = {'stream_name': self.CHANNEL_NAME,
-                   'content_type': 'application/x-www-form-urlencoded'}
-    result = self.client_post(self.url, 'unknown_action', **post_params)
-
-    # check that we got the expected error message
-    self.assert_json_error(result, "Unknown WordPress webhook action: WordPress action")
-```
-
-In a normal test, `check_webhook` would handle all the setup
-and then check that the incoming webhook's response matches the expected result. If
-the webhook returns an error, the test fails. Instead, explicitly do the
-setup it would have done, and check the result yourself.
-
-Here, `subscribe_to_stream` is a test helper that uses `TEST_USER_EMAIL` and
-`CHANNEL_NAME` (attributes from the base class) to register the user to receive
-messages in the given channel. If the channel doesn't exist, it creates it.
-
-`client_post`, another helper, performs the HTTP POST that calls the incoming
-webhook. As long as `self.url` is correct, you don't need to construct the webhook
-URL yourself. (In most cases, it is.)
-
-`assert_json_error` then checks if the result matches the expected error.
-If you had used `check_webhook`, it would have called
-`send_webhook_payload`, which checks the result with `assert_json_success`.
-
-### Custom query parameters
-
-Custom arguments passed in URL query parameters work as expected in the webhook
-code, but require special handling in tests.
-
-For example, here is the definition of a webhook function that gets both `stream`
-and `topic` from the query parameters:
-
-```python
-@typed_endpoint
-def api_querytest_webhook(request: HttpRequest, user_profile: UserProfile,
-                          payload: Annotated[str, ApiParamConfig(argument_type_is_body=True)],
-                          stream: str = "test",
-                          topic: str= "Default Alert":
-```
-
-In actual use, you might configure the 3rd party service to call your Zulip
-integration with a URL like this:
-
-```
-http://myhost/api/v1/external/querytest?api_key=abcdefgh&stream=alerts&topic=queries
-```
-
-It provides values for `stream` and `topic`, and the webhook can get those
-using `@typed_endpoint` without any special handling. How does this work in a test?
-
-The new attribute `TOPIC` exists only in our class so far. In order to
-construct a URL with a query parameter for `topic`, you can pass the
-attribute `TOPIC` as a keyword argument to `build_webhook_url`, like so:
-
-```python
-class QuerytestHookTests(WebhookTestCase):
-
-    CHANNEL_NAME = 'querytest'
-    TOPIC = "Default topic"
-    URL_TEMPLATE = "/api/v1/external/querytest?api_key={api_key}&stream={stream}"
-    FIXTURE_DIR_NAME = 'querytest'
-
-    def test_querytest_test_one(self) -> None:
-        # construct the URL used for this test
-        self.TOPIC = "Query test"
-        self.url = self.build_webhook_url(topic=self.TOPIC)
-
-        # define the expected message contents
-        expected_topic = "Query test"
-        expected_message = "This is a test of custom query parameters."
-
-        self.check_webhook('test_one', expected_topic, expected_message,
-                                          content_type="application/x-www-form-urlencoded")
-```
-
-You can also override `get_body` or `get_payload` if your test data
-needs to be constructed in an unusual way.
-
-For more, see the definition for the base class, `WebhookTestCase`
-in `zerver/lib/test_classes.py`, or just grep for examples.
-
-
-### Custom HTTP event-type headers
-
-Some third-party services set a custom HTTP header to indicate the event type that
-generates a particular payload. To extract such headers, we recommend using the
-`validate_extract_webhook_http_header` function in `zerver/lib/webhooks/common.py`,
-like so:
-
-```python
-event = validate_extract_webhook_http_header(request, header, integration_name)
-```
-
-`request` is the `HttpRequest` object passed to your main webhook function. `header`
-is the name of the custom header you'd like to extract, such as `X-Event-Key`, and
-`integration_name` is the name of the third-party service in question, such as
-`GitHub`.
-
-Because such headers are how some integrations indicate the event types of their
-payloads, the absence of such a header usually indicates a configuration
-issue, where one either entered the URL for a different integration, or happens to
-be running an older version of the integration that doesn't set that header.
-
-If the requisite header is missing, this function sends a direct message to the
-owner of the webhook bot, notifying them of the missing header.
-
-### Handling unexpected webhook event types
-
-Many third-party services have dozens of different event types. In
-some cases, we may choose to explicitly ignore specific events. In
-other cases, there may be events that are new or events that we don't
-know about. In such cases, we recommend raising
-`UnsupportedWebhookEventTypeError` (found in `zerver/lib/exceptions.py`),
-with a string describing the unsupported event type, like so:
-
-```
-raise UnsupportedWebhookEventTypeError(event_type)
-```
-
-## Related articles
-
-* [Integrations overview](/api/integrations-overview)
-* [Incoming webhook integrations](/api/incoming-webhooks-overview)

api_docs/index.md

@@ -1,26 +0,0 @@
-# The Zulip API
-
-Zulip's APIs allow you to integrate other services with Zulip.  This
-guide should help you find the API you need:
-
-* First, check if the tool you'd like to integrate with Zulip
-  [already has a native integration](/integrations/).
-* Next, check if [Zapier](https://zapier.com/apps) or
-  [IFTTT](https://ifttt.com/search) has an integration.
-  [Zulip's Zapier integration](/integrations/doc/zapier) and
-  [Zulip's IFTTT integration](/integrations/doc/ifttt) often allow
-  integrating a new service with Zulip without writing any code.
-* If you'd like to send content into Zulip, you can
-  [write a native incoming webhook integration](/api/incoming-webhooks-overview)
-  or use [Zulip's API for sending messages](/api/send-message).
-* If you're building an interactive bot that reacts to activity inside
-  Zulip, you'll want to look at Zulip's
-  [Python framework for interactive bots](/api/running-bots) or
-  [Zulip's real-time events API](/api/get-events).
-
-And if you still need to build your own integration with Zulip, check out
-the full [REST API](/api/rest), generally starting with
-[installing the API client bindings](/api/installation-instructions).
-
-In case you already know how you want to build your integration and you're
-just looking for an API key, we've got you covered [here](/api/api-keys).

api_docs/integrations-overview.md

@@ -1,139 +0,0 @@
-# Integrations overview
-
-Integrations let you connect Zulip with other products. For example, you can get
-notification messages in Zulip when an issue in your tracker is updated, or for
-alerts from your monitoring tool.
-
-Zulip offers [over 120 native integrations](/integrations/), which take
-advantage of Zulip's [topics](/help/introduction-to-topics) to organize
-notification messages. Additionally, thousands of integrations are available
-through [Zapier](https://zapier.com/apps) and [IFTTT](https://ifttt.com/search).
-You can also [connect any webhook designed to work with
-Slack](/integrations/doc/slack_incoming) to Zulip.
-
-If you don't find an integration you need, you can:
-
-- [Write your own integration](#write-your-own-integration). You can [submit a
-pull
-request](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html)
-to get your integration merged into the main Zulip repository.
-
-- [File an issue](https://github.com/zulip/zulip/issues/new/choose) to request
-  an integration (if it's a nice-to-have).
-
-- [Contact Zulip Sales](mailto:sales@zulip.com) to inquire about a custom
-  development contract.
-
-## Set up an integration
-
-### Native integrations
-
-{start_tabs}
-
-1. [Search Zulip's integrations](/integrations/) for the product you'd like to
-   connect to Zulip.
-
-1. Click on the card for the product, and follow the instructions on the page.
-
-{end_tabs}
-
-### Integrate via Zapier or IFTTT
-
-If you don't see a native Zulip integration, you can access thousands of
-additional integrations through [Zapier](https://zapier.com/apps) and
-[IFTTT](https://ifttt.com/search).
-
-{start_tabs}
-
-1. Search [Zapier](https://zapier.com/apps) or [IFTTT](https://ifttt.com/search)
-for the product you'd like to connect to Zulip.
-
-1. Follow the integration instructions for [Zapier](/integrations/doc/zapier) or
-   [IFTTT](/integrations/doc/ifttt).
-
-{end_tabs}
-
-### Integrate via Slack-compatible webhook API
-
-Zulip can process incoming webhook messages written to work with [Slack's
-webhook API](https://api.slack.com/messaging/webhooks). This makes it easy to
-quickly move your integrations when [migrating your
-organization](/help/import-from-slack) from Slack to Zulip, or integrate any
-product that has a Slack webhook integration with Zulip .
-
-!!! warn ""
-
-     **Note:** In the long term, the recommended approach is to use
-     Zulip's native integrations, which take advantage of Zulip's topics.
-     There may also be some quirks when Slack's formatting system is
-     translated into Zulip's.
-
-{start_tabs}
-
-1. [Create a bot](/help/add-a-bot-or-integration) for the Slack-compatible
-   webhook. Make sure that you select **Incoming webhook** as the **Bot type**.
-
-1. Decide where to send Slack-compatible webhook notifications, and [generate
-   the integration URL](https://zulip.com/help/generate-integration-url).
-
-1. Use the generated URL anywhere you would use a Slack webhook.
-
-{end_tabs}
-
-### Integrate via email
-
-If the product you'd like to integrate can send email notifications, you can
-[send those emails to a Zulip channel](/help/message-a-channel-by-email). The
-email subject will become the Zulip topic, and the email body will become the
-Zulip message.
-
-For example, you can configure your personal GitHub notifications to go to a
-Zulip channel rather than your email inbox. Notifications for each issue or pull
-request will be grouped into a single topic.
-
-## Write your own integration
-
-You can write your own Zulip integrations using the well-documented APIs below.
-For example, if your company develops software, you can create a custom
-integration to connect your product to Zulip.
-
-If you need help, best-effort community support is available in the [Zulip
-development community](https://zulip.com/development-community/). To inquire
-about options for custom development, [contact Zulip
-Sales](mailto:sales@zulip.com).
-
-### Sending content into Zulip
-
-* If the third-party service supports outgoing webhooks, you likely want to
-  build an [incoming webhook integration](/api/incoming-webhooks-overview).
-
-* If it doesn't, you may want to write a
-  [script or plugin integration](/api/non-webhook-integrations).
-
-* The [`zulip-send` tool](/api/send-message) makes it easy to send Zulip
-  messages from shell scripts.
-
-* Finally, you can
-  [send messages using Zulip's API](/api/send-message), with bindings for
-  Python, JavaScript and [other languages](/api/client-libraries).
-
-### Sending and receiving content
-
-* To react to activity inside Zulip, look at Zulip's
-  [Python framework for interactive bots](/api/running-bots) or
-  [Zulip's real-time events API](/api/get-events).
-
-* If what you want isn't covered by the above, check out the full
-  [REST API](/api/rest). The web, mobile, desktop, and terminal apps are
-  built on top of this API, so it can do anything a human user can do. Most
-  but not all of the endpoints are documented on this site; if you need
-  something that isn't there check out Zulip's
-  [REST endpoints](https://github.com/zulip/zulip/blob/main/zproject/urls.py).
-
-## Related articles
-
-* [Bots overview](/help/bots-overview)
-* [Set up integrations](/help/set-up-integrations)
-* [Add a bot or integration](/help/add-a-bot-or-integration)
-* [Generate integration URL](/help/generate-integration-url)
-* [Request an integration](/help/request-an-integration)

api_docs/message-formatting.md

@@ -1,364 +0,0 @@
-# Message formatting
-
-Zulip supports an extended version of Markdown for messages, as well as
-some HTML level special behavior. The Zulip help center article on [message
-formatting](/help/format-your-message-using-markdown) is the primary
-documentation for Zulip's markup features. This article is currently a
-changelog for updates to these features.
-
-The [render a message](/api/render-message) endpoint can be used to get
-the current HTML version of any Markdown syntax for message content.
-
-## Code blocks
-
-**Changes**: As of Zulip 4.0 (feature level 33), [code blocks][help-code]
-can have a `data-code-language` attribute attached to the outer HTML
-`div` element, which records the programming language that was selected
-for syntax highlighting. This field is used in the
-[playgrounds][help-playgrounds] feature for code blocks.
-
-## Global times
-
-**Changes**: In Zulip 3.0 (feature level 8), added [global time
-mentions][help-global-time] to supported Markdown message formatting
-features.
-
-## Links to channels, topics, and messages
-
-Zulip's markup supports special readable Markdown syntax for [linking
-to channels, topics, and messages](/help/link-to-a-message-or-conversation).
-
-Sample HTML formats are as follows:
-``` html
-<!-- Syntax: #**announce** -->
-<a class="stream" data-stream-id="9"
-  href="/#narrow/channel/9-announce">
- #announce
-</a>
-
-<!-- Syntax: #**announce>Zulip updates** -->
-<a class="stream-topic" data-stream-id="9"
-  href="/#narrow/channel/9-announce/topic/Zulip.20updates/with/214">
- #announce &gt; Zulip updates
-</a>
-
-<!-- Syntax: #**announce>Zulip updates**
-     Generated only if topic had no messages or the link was rendered
-     before Zulip 10.0 (feature level 347) -->
-<a class="stream-topic" data-stream-id="9"
-  href="/#narrow/channel/9-announce/topic/Zulip.20updates">
- #announce &gt; Zulip updates
-</a>
-
-<!-- Syntax: #**announce>Zulip updates@214** -->
-<a class="message-link"
-  href="/#narrow/channel/9-announce/topic/Zulip.20updates/near/214">
- #announce &gt; Zulip updates @ 💬
-</a>
-```
-
-The `near` and `with` operators are documented in more detail in the
-[search and URL documentation](/api/construct-narrow). When rendering
-topic links with the `with` operator, the code doing the rendering may
-pick the ID arbitrarily among messages accessible to the client and/or
-acting user at the time of rendering. Currently, the server chooses
-the message ID to use for `with` operators as the oldest message ID in
-the topic accessible to the user who wrote the message. In channels
-with protected history, this means the same Markdown syntax may be
-rendered differently for users who joined at different times.
-
-The older stream/topic link elements include a `data-stream-id`, which
-historically was used in order to display the current channel name if
-the channel had been renamed. That field is **deprecated**, because
-displaying an updated value for the most common forms of this syntax
-requires parsing the URL to get the topic to use anyway.
-
-When a topic is an empty string, it is replaced with
-`realm_empty_topic_display_name` found in the [`POST /register`](/api/register-queue)
-response and wrapped with the `<em>` tag.
-
-Sample HTML formats with `"realm_empty_topic_display_name": "general chat"`
-are as follows:
-```html
-<!-- Syntax: #**announce>** -->
-<a class="stream-topic" data-stream-id="9"
-  href="/#narrow/channel/9-announce/topic/with/214">
- #announce &gt; <em>general chat</em>
-</a>
-
-<!-- Syntax: #**announce>**
-     Generated only if topic had no messages or the link was rendered
-     before Zulip 10.0 (feature level 347) -->
-<a class="stream-topic" data-stream-id="9"
-  href="/#narrow/channel/9-announce/topic/">
- #announce &gt; <em>general chat</em>
-</a>
-
-<!-- Syntax: #**announce>@214** -->
-<a class="message-link"
-  href="/#narrow/channel/9-announce/topic//near/214">
- #announce &gt; <em>general chat</em> @ 💬
-</a>
-```
-
-**Changes**: Before Zulip 10.0 (feature level 347), the `with` field
-was never used in topic link URLs generated by the server; the markup
-currently used only for empty topics was used for all topic links.
-
-Before Zulip 10.0 (feature level 346), empty string
-was not a valid topic name in syntaxes for linking to topics and
-messages.
-
-In Zulip 10.0 (feature level 319), added Markdown syntax
-for linking to a specific message in a conversation. Declared the
-`data-stream-id` field to be deprecated as detailed above.
-
-## Image previews
-
-When a Zulip message is sent linking to an uploaded image, Zulip will
-generate an image preview element with the following format.
-
-``` html
-<div class="message_inline_image">
-    <a href="/user_uploads/path/to/image.png" title="image.png">
-        <img data-original-dimensions="1920x1080"
-          data-original-content-type="image/png"
-          src="/user_uploads/thumbnail/path/to/image.png/840x560.webp">
-    </a>
-</div>
-```
-
-If the server has not yet generated thumbnails for the image yet at
-the time the message is sent, the `img` element will be a temporary
-loading indicator image and have the `image-loading-placeholder`
-class, which clients can use to identify loading indicators and
-replace them with a more native loading indicator element if
-desired. For example:
-
-``` html
-<div class="message_inline_image">
-    <a href="/user_uploads/path/to/image.png" title="image.png">
-        <img class="image-loading-placeholder"
-          data-original-dimensions="1920x1080"
-          data-original-content-type="image/png"
-          src="/path/to/spinner.png">
-    </a>
-</div>
-```
-
-Once the server has a working thumbnail, such messages will be updated
-via an `update_message` event, with the `rendering_only: true` flag
-(telling clients not to adjust message edit history), with appropriate
-adjusted `rendered_content`. A client should process those events by
-just using the updated rendering. If thumbnailing failed, the same
-type of event will edit the message's rendered form to remove the
-image preview element, so no special client-side logic should be
-required to process such errors.
-
-Note that in the uncommon situation that the thumbnailing system is
-backlogged, an individual message containing multiple image previews
-may be re-rendered multiple times as each image finishes thumbnailing
-and triggers a message update.
-
-Clients are recommended to do the following when processing image
-previews:
-
-- Clients that would like to use the image's aspect ratio to lay out
-  one or more images in the message feed may use the
-  `data-original-dimensions` attribute, which is present even if the
-  image is a placeholder spinner.  This attribute encodes the
-  dimensions of the original image as `{width}x{height}`.  These
-  dimensions are for the image as rendered, _after_ any EXIF rotation
-  and mirroring has been applied.
-- If the client would like to control the thumbnail resolution used,
-  it can replace the final section of the URL (`840x560.webp` in the
-  example above) with the `name` of its preferred format from the set
-  of supported formats provided by the server in the
-  `server_thumbnail_formats` portion of the `register`
-  response. Clients should not make any assumptions about what format
-  the server will use as the "default" thumbnail resolution, as it may
-  change over time.
-- Download button type elements should provide the original image
-  (encoded via the `href` of the containing `a` tag).
-- The content-type of the original image is provided on a
-  `data-original-content-type` attribute, so clients can decide if
-  they are capable of rendering the original image.
-- For images whose formats which are not widely-accepted by browsers
-  (e.g., HEIC and TIFF), the image may contain a
-  `data-transcoded-image` attribute, which specifies a high-resolution
-  thumbnail format which clients may use instead of the original
-  image.
-- Lightbox elements for viewing an image should be designed to
-  immediately display any already-downloaded thumbnail while fetching
-  the original-quality image or an appropriate higher-quality
-  thumbnail from the server, to be transparently swapped in once it is
-  available. Clients that would like to size the lightbox based on the
-  size of the original image can use the `data-original-dimensions`
-  attribute, as described above.
-- Animated images will have a `data-animated` attribute on the `img`
-  tag. As detailed in `server_thumbnail_formats`, both animated and
-  still images are available for clients to use, depending on their
-  preference. See, for example, the [web setting][help-previews]
-  to control whether animated images are autoplayed in the message
-  feed.
-- Clients should not assume that the requested format is the format
-  that they will receive; in rare cases where the client has an
-  out-of-date list of `server_thumbnail_formats`, the server will
-  provide an approximation of the client's requested format.  Because
-  of this, clients should not assume that the pixel dimensions or file
-  format match what they requested.
-- No other processing of the URLs is recommended.
-
-**Changes**: In Zulip 10.0 (feature level 336), added
-`data-original-content-type` attribute to convey the type of the
-original image, and optional `data-transcoded-image` attribute for
-images with formats which are not widely supported by browsers.
-
-**Changes**: In Zulip 9.2 (feature levels 278-279, and 287+), added
-`data-original-dimensions` to the `image-loading-placeholder` spinner
-images, containing the dimensions of the original image.
-
-In Zulip 9.0 (feature level 276), added `data-original-dimensions`
-attribute to images that have been thumbnailed, containing the
-dimensions of the full-size version of the image. Thumbnailing itself
-was reintroduced at feature level 275.
-
-Previously, with the exception of Zulip servers that used the beta
-Thumbor-based implementation years ago, all image previews in Zulip
-messages were not thumbnailed; the `a` tag and the `img` tag would both
-point to the original image.
-
-Clients that correctly implement the current API should handle
-Thumbor-based older thumbnails correctly, as long as they do not
-assume that `data-original-dimensions` is present. Clients should not
-assume that messages sent prior to the introduction of thumbnailing
-have been re-rendered to use the new format or have thumbnails
-available.
-
-## Mentions and silent mentions
-
-Zulip markup supports [mentioning](/help/mention-a-user-or-group)
-users, user groups, and a few special "wildcard" mentions (the three
-spellings of a channel wildcard mention: `@**all**`, `@**everyone**`,
-`@**channel**` and the topic wildcard mention `@**topic**`).
-
-Mentions result in a message being highlighted for the target user(s),
-both in the UI and in notifications, and may also result in the target
-user(s) following the conversation, [depending on their
-settings](/help/follow-a-topic#follow-topics-where-you-are-mentioned).
-
-Silent mentions of users or groups have none of those side effects,
-but nonetheless uniquely identify the user or group
-identified. (There's no such thing as a silent wildcard mention).
-
-Permissions for mentioning users work as follows:
-
-- Any user can mention any other user, though mentions by [muted
-users](/help/mute-a-user) are automatically marked as read and thus do
-not trigger notifications or otherwise get highlighted like unread
-mentions.
-
-- Wildcard mentions are permitted except where [organization-level
-restrictions](/help/restrict-wildcard-mentions) apply.
-
-- User groups can be mentioned if and only if the acting user is in
-the `can_mention_group` group for that group. All user groups can be
-silently mentioned by any user.
-
-- System groups, when (silently) mentioned, should be displayed using
-their description, not their `role:nobody` style API names; see the
-main [system group
-documentation](/api/group-setting-values#system-groups) for
-details. System groups can only be silently mentioned right now,
-because they happen to all use the empty `Nobody` group for
-`can_mention_group`; clients should just use `can_mention_group` to
-determine which groups to offer in typeahead in similar contexts.
-
-- Requests to send or edit a message that are impermissible due to
-including a mention where the acting user does not have permission to
-mention the target will return an error. Mention syntax that does not
-correspond to a real user or group is ignored.
-
-Sample markup for `@**Example User**`:
-
-``` html
-<span class="user-mention" data-user-id="31">@Example User</span>
-```
-
-Sample markup for `@_**Example User**`:
-
-``` html
-<span class="user-mention silent" data-user-id="31">Example User</span>
-```
-
-Sample markup for `@**topic**`:
-
-``` html
-<span class="topic-mention">@topic</span>
-```
-
-Sample markup for `@**channel**`:
-
-``` html
-<span class="user-mention channel-wildcard-mention"
-  data-user-id="*">@channel</span>
-```
-
-Sample markup for `@*support*`, assuming "support" is a valid group:
-``` html
-<span class="user-group-mention"
-  data-user-group-id="17">@support</span>
-```
-
-Sample markup for `@_*support*`, assuming "support" is a valid group:
-``` html
-<span class="user-group-mention silent"
-  data-user-group-id="17">support</span>
-```
-
-Sample markup for `@_*role:administrators*`:
-``` html
-<span class="user-group-mention silent"
-  data-user-group-id="5">Administrators</span>
-```
-
-When processing mentions, clients should look up the user or group
-referenced by ID, and update the textual name for the mention to the
-current name for the user or group with that ID. Note that for system
-groups, this requires special logic to look up the user-facing name
-for that group; see [system
-groups](/api/group-setting-values#system-groups) for details.
-
-**Changes**: Prior to Zulip 10.0 (feature level 333), it was not
-possible to silently mention [system
-groups](/api/group-setting-values#system-groups).
-
-In Zulip 9.0 (feature level 247), `channel` was added to the supported
-[wildcard][help-mention-all] options used in the
-[mentions][help-mentions] Markdown message formatting feature.
-
-## Spoilers
-
-**Changes**: In Zulip 3.0 (feature level 15), added
-[spoilers][help-spoilers] to supported Markdown message formatting
-features.
-
-## Removed features
-
-**Changes**: In Zulip 4.0 (feature level 24), the rarely used `!avatar()`
-and `!gravatar()` markup syntax, which was never documented and had an
-inconsistent syntax, were removed.
-
-## Related articles
-
-* [Markdown formatting](/help/format-your-message-using-markdown)
-* [Send a message](/api/send-message)
-* [Render a message](/api/render-message)
-
-[help-code]: /help/code-blocks
-[help-playgrounds]: /help/code-blocks#code-playgrounds
-[help-spoilers]: /help/spoilers
-[help-global-time]: /help/global-times
-[help-mentions]: /help/mention-a-user-or-group
-[help-mention-all]: /help/mention-a-user-or-group#mention-everyone-on-a-channel
-[help-previews]: /help/image-video-and-website-previews#configure-how-animated-images-are-played

api_docs/rest-error-handling.md

@@ -1,64 +0,0 @@
-# Error handling
-
-Zulip's API will always return a JSON format response.
-The HTTP status code indicates whether the request was successful
-(200 = success, 4xx = user error, 5xx = server error).
-
-Every response, both success and error responses, will contain at least
-two keys:
-
-- `msg`: an internationalized, human-readable error message string.
-
-- `result`: either `"error"` or `"success"`, which is redundant with the
-  HTTP status code, but is convenient when print debugging.
-
-Every error response will also contain an additional key:
-
-- `code`: a machine-readable error string, with a default value of
-  `"BAD_REQUEST"` for general errors.
-
-Clients should always check `code`, rather than `msg`, when looking for
-specific error conditions. The string values for `msg` are
-internationalized (e.g., the server will send the error message
-translated into French if the user has a French locale), so checking
-those strings will result in buggy code.
-
-!!! tip ""
-
-     If a client needs information that is only present in the string value
-     of `msg` for a particular error response, then the developers
-     implementing the client should [start a conversation here][api-design]
-     in order to discuss getting a specific error `code` and/or relevant
-     additional key/value pairs for that error response.
-
-In addition to the keys described above, some error responses will
-contain other keys with further details that are useful for clients. The
-specific keys present depend on the error `code`, and are documented at
-the API endpoints where these particular errors appear.
-
-**Changes**: Before Zulip 5.0 (feature level 76), all error responses
-did not contain a `code` key, and its absence indicated that no specific
-error `code` had been allocated for that error.
-
-## Common error responses
-
-Documented below are some error responses that are common to many
-endpoints:
-
-{generate_code_example|/rest-error-handling:post|fixture}
-
-## Ignored Parameters
-
-In JSON success responses, all Zulip REST API endpoints may return
-an array of parameters sent in the request that are not supported
-by that specific endpoint.
-
-While this can be expected, e.g., when sending both current and legacy
-names for a parameter to a Zulip server of unknown version, this often
-indicates either a bug in the client implementation or an attempt to
-configure a new feature while connected to an older Zulip server that
-does not support said feature.
-
-{generate_code_example|/settings:patch|fixture}
-
-[api-design]: https://chat.zulip.org/#narrow/channel/378-api-design

api_docs/roles-and-permissions.md

@@ -1,125 +0,0 @@
-# Roles and permissions
-
-Zulip offers several levels of permissions based on a
-[user's role](/help/user-roles) in a Zulip organization.
-
-Here are some important details to note when working with these
-roles and permissions in Zulip's API:
-
-## A user's role
-
-A user's account data include a `role` property, which contains the
-user's role in the Zulip organization. These roles are encoded as:
-
-* Organization owner: 100
-
-* Organization administrator: 200
-
-* Organization moderator: 300
-
-* Member: 400
-
-* Guest: 600
-
-User account data also include these boolean properties that duplicate
-the related roles above:
-
-* `is_owner` specifying whether the user is an organization owner.
-
-* `is_admin` specifying whether the user is an organization administrator.
-
-* `is_guest` specifying whether the user is a guest user.
-
-These are intended as conveniences for simple clients, and clients
-should prefer using the `role` field, since only that one is updated
-by the [events API](/api/get-events).
-
-Note that [`POST /register`](/api/register-queue) also returns an
-`is_moderator` boolean property specifying whether the current user is
-an organization moderator.
-
-Additionally, user account data include an `is_billing_admin` property
-specifying whether the user is a billing administrator for the Zulip
-organization, which is not related to one of the roles listed above,
-but rather allows for specific permissions related to billing
-administration in [paid Zulip Cloud plans](https://zulip.com/plans/).
-
-### User account data in the API
-
-Endpoints that return the user account data / properties mentioned
-above are:
-
-* [`GET /users`](/api/get-users)
-
-* [`GET /users/{user_id}`](/api/get-user)
-
-* [`GET /users/{email}`](/api/get-user-by-email)
-
-* [`GET /users/me`](/api/get-own-user)
-
-* [`GET /events`](/api/get-events)
-
-* [`POST /register`](/api/register-queue)
-
-Note that the [`POST /register` endpoint](/api/register-queue) returns
-the above boolean properties to describe the role of the current user,
-when `realm_user` is present in `fetch_event_types`.
-
-Additionally, the specific events returned by the
-[`GET /events` endpoint](/api/get-events) containing data related
-to user accounts and roles are the [`realm_user` add
-event](/api/get-events#realm_user-add), and the
-[`realm_user` update event](/api/get-events#realm_user-update).
-
-## Permission levels
-
-Many areas of Zulip are customizable by the roles
-above, such as (but not limited to) [restricting message editing and
-deletion](/help/restrict-message-editing-and-deletion) and various
-permissions for different [channel types](/help/channel-permissions).
-The potential permission levels are:
-
-* Everyone / Any user including Guests (least restrictive)
-
-* Members
-
-* Full members
-
-* Moderators
-
-* Administrators
-
-* Owners
-
-* Nobody (most restrictive)
-
-These permission levels and policies in the API are designed to be
-cutoffs in that users with the specified role and above have the
-specified ability or access. For example, a permission level documented
-as 'moderators only' includes organization moderators, administrators,
-and owners.
-
-Note that specific settings and policies in the Zulip API that use these
-permission levels will likely support a subset of those listed above.
-
-## Group-based permissions
-
-Some settings have been migrated to a more flexible system based on
-[user groups](/api/group-setting-values).
-
-## Determining if a user is a full member
-
-When a Zulip organization has set up a [waiting period before new members
-turn into full members](/help/restrict-permissions-of-new-members),
-clients will need to determine if a user's account has aged past the
-organization's waiting period threshold.
-
-The `realm_waiting_period_threshold`, which is the number of days until
-a user's account is treated as a full member, is returned by the
-[`POST /register` endpoint](/api/register-queue) when `realm` is present
-in `fetch_event_types`.
-
-Clients can compare the `realm_waiting_period_threshold` to a user
-accounts's `date_joined` property, which is the time the user account
-was created, to determine if a user has the permissions of a full
-member or a new member.

api_docs/running-bots.md

@@ -1,74 +0,0 @@
-# Interactive bots
-
-Zulip's API has a powerful framework for interactive bots that react
-to messages in Zulip.
-
-## Running a bot
-
-This guide will show you how to run an existing Zulip bot
-found in [zulip_bots/bots](
-https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots).
-
-You'll need:
-
-* An account in a Zulip organization
-  (e.g., [the Zulip development community](https://zulip.com/development-community/),
-  `<yourSubdomain>.zulipchat.com`, or a Zulip organization on your own
-  [development](https://zulip.readthedocs.io/en/latest/development/overview.html) or
-  [production](https://zulip.readthedocs.io/en/latest/production/install.html) server).
-* A computer where you're running the bot from.
-
-**Note: Please be considerate when testing experimental bots on public servers such as chat.zulip.org.**
-
-{start_tabs}
-
-1. [Create a bot](/help/add-a-bot-or-integration), making sure to select
-   **Generic bot** as the **Bot type**.
-
-1. [Download the bot's `zuliprc` file](/api/configuring-python-bindings#download-a-zuliprc-file).
-
-1. Use the following command to install the
-   [`zulip_bots` Python package](https://pypi.org/project/zulip-bots/):
-
-        pip3 install zulip_bots
-
-1. Use the following command to start the bot process *(replacing
-   `~/path/to/zuliprc` with the path to the `zuliprc` file you downloaded above)*:
-
-        zulip-run-bot <bot-name> --config-file ~/path/to/zuliprc
-
-1. Check the output of the command above to make sure your bot is running.
-   It should include the following line:
-
-        INFO:root:starting message handling...
-
-1. Test your setup by [starting a new direct message](/help/starting-a-new-direct-message)
-   with the bot or [mentioning](/help/mention-a-user-or-group) the bot on a channel.
-
-!!! tip ""
-
-    To use the latest development version of the `zulip_bots` package, follow
-    [these steps](writing-bots#installing-a-development-version-of-the-zulip-bots-package).
-
-{end_tabs}
-
-You can now play around with the bot and get it configured the way you
-like.  Eventually, you'll probably want to run it in a production
-environment where it'll stay up, by [deploying](/api/deploying-bots) it on a
-server using the Zulip Botserver.
-
-## Common problems
-
-* My bot won't start
-    * Ensure that your API config file is correct (download the config file from the server).
-    * Ensure that your bot script is located in `zulip_bots/bots/<my-bot>/`
-    * Are you using your own Zulip development server? Ensure that you run your bot outside
-      the Vagrant environment.
-    * Some bots require Python 3. Try switching to a Python 3 environment before running
-      your bot.
-
-## Related articles
-
-* [Non-webhook integrations](/api/non-webhook-integrations)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)

api_docs/send-message.md

@@ -1,77 +0,0 @@
-{generate_api_header(/messages:post)}
-
-## Usage examples
-
-{start_tabs}
-
-{generate_code_example(python)|/messages:post|example}
-
-{generate_code_example(javascript)|/messages:post|example}
-
-{tab|curl}
-
-``` curl
-# For channel messages
-curl -X POST {{ api_url }}/v1/messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=stream \
-    --data-urlencode 'to="Denmark"' \
-    --data-urlencode topic=Castle \
-    --data-urlencode 'content=I come not, friends, to steal away your hearts.'
-
-# For direct messages
-curl -X POST {{ api_url }}/v1/messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=direct \
-    --data-urlencode 'to=[9]' \
-    --data-urlencode 'content=With mirth and laughter let old wrinkles come.'
-```
-
-{tab|zulip-send}
-
-You can use `zulip-send`
-(available after you `pip install zulip`) to easily send Zulips from
-the command-line, providing the message content via STDIN.
-
-```bash
-# For channel messages
-zulip-send --stream Denmark --subject Castle \
-    --user othello-bot@example.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
-
-# For direct messages
-zulip-send hamlet@example.com \
-    --user othello-bot@example.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
-```
-
-#### Passing in the message on the command-line
-
-If you'd like, you can also provide the message on the command-line with the
-`-m` or `--message` flag, as follows:
-
-
-```bash
-zulip-send --stream Denmark --subject Castle \
-    --message 'I come not, friends, to steal away your hearts.' \
-    --user othello-bot@example.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
-```
-
-You can omit the `user` and `api-key` parameters if you have a `~/.zuliprc`
-file.
-
-{end_tabs}
-
-## Parameters
-
-{generate_api_arguments_table|zulip.yaml|/messages:post}
-
-{generate_parameter_description(/messages:post)}
-
-## Response
-
-{generate_return_values_table|zulip.yaml|/messages:post}
-
-{generate_response_description(/messages:post)}
-
-#### Example response(s)
-
-{generate_code_example|/messages:post|fixture}

api_docs/sidebar_index.md

@@ -1,29 +0,0 @@
-## Integrations
-
-* [Overview](/api/integrations-overview)
-* [Incoming webhook integrations](/api/incoming-webhooks-overview)
-* [Hello world walkthrough](/api/incoming-webhooks-walkthrough)
-* [Non-webhook integrations](/api/non-webhook-integrations)
-
-## Interactive bots (beta)
-
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)
-* [Outgoing webhooks](/api/outgoing-webhooks)
-
-## REST API
-
-* [Overview](/api/rest)
-* [Installation instructions](/api/installation-instructions)
-* [API keys](/api/api-keys)
-* [Configuring the Python bindings](/api/configuring-python-bindings)
-* [HTTP headers](/api/http-headers)
-* [Error handling](/api/rest-error-handling)
-* [Roles and permissions](/api/roles-and-permissions)
-* [Group-setting values](/api/group-setting-values)
-* [Message formatting](/api/message-formatting)
-* [Client libraries](/api/client-libraries)
-* [API changelog](/api/changelog)
-
-{!rest-endpoints.md!}

api_docs/writing-bots.md

@@ -1,614 +0,0 @@
-# Writing interactive bots
-
-This guide is about writing and testing interactive bots. We assume
-familiarity with our [guide for running bots](running-bots).
-
-On this page you'll find:
-
-* A step-by-step
-  [guide](#installing-a-development-version-of-the-zulip-bots-package)
-  on how to set up a development environment for writing bots with all
-  of our nice tooling to make it easy to write and test your work.
-* A [guide](#writing-a-bot) on writing a bot.
-* A [guide](#adding-a-bot-to-zulip) on adding a bot to Zulip.
-* A [guide](#testing-a-bots-output) on testing a bot's output.
-* [Documentation](#bot-api) of the bot API.
-* Common [problems](#common-problems) when developing/running bots and their solutions.
-
-## Installing a development version of the Zulip bots package
-
-{start_tabs}
-
-1. Clone the [python-zulip-api](https://github.com/zulip/python-zulip-api)
-   repository:
-
-    ```
-    git clone https://github.com/zulip/python-zulip-api.git
-    ```
-
-1. Navigate into your cloned repository:
-
-    ```
-    cd python-zulip-api
-    ```
-
-1. Install all requirements in a Python virtualenv:
-
-    ```
-    python3 ./tools/provision
-    ```
-
-1. Run the command provided in the final output of the `provision` process to
-   enter the new virtualenv. The command will be of the form `source .../activate`.
-
-1. You should now see the name of your virtualenv preceding your prompt (e.g.,
-   `(zulip-api-py3-venv)`).
-
-!!! tip ""
-
-    `provision` installs the `zulip`, `zulip_bots`, and
-    `zulip_botserver` packages in developer mode. This enables you to
-    modify these packages and then run your modified code without
-    having to first re-install the packages or re-provision.
-
-{end_tabs}
-
-## Writing a bot
-
-The tutorial below explains the structure of a bot `<my-bot>.py`,
-which is the only file you need to create for a new bot. You
-can use this as boilerplate code for developing your own bot.
-
-Every bot is built upon this structure:
-
-```python
-class MyBotHandler(object):
-    '''
-    A docstring documenting this bot.
-    '''
-
-    def usage(self):
-        return '''Your description of the bot'''
-
-    def handle_message(self, message, bot_handler):
-        # add your code here
-
-handler_class = MyBotHandler
-```
-
-* The class name (in this case *MyBotHandler*) can be defined by you
-  and should match the name of your bot. To register your bot's class,
-  adjust the last line `handler_class = MyBotHandler` to match your
-  class name.
-
-* Every bot needs to implement the functions
-    * `usage(self)`
-    * `handle_message(self, message, bot_handler)`
-
-* These functions are documented in the [next section](#bot-api).
-
-## Adding a bot to Zulip
-
-Zulip's bot system resides in the [python-zulip-api](
-https://github.com/zulip/python-zulip-api) repository.
-
-The structure of the bots ecosystem looks like the following:
-
-```
-zulip_bots
-└───zulip_bots
-    ├───bots
-    │   ├───bot1
-    │   └───bot2
-    │       │
-    │       ├───bot2.py
-    │       ├───bot2.conf
-    │       ├───doc.md
-    │       ├───requirements.txt
-    │       ├───test_bot2.py
-    │       ├───assets
-    │       │   │
-    │       │   └───pic.png
-    │       ├───fixtures
-    │       │   │
-    │       │   └───test1.json
-    │       └───libraries
-    │           │
-    │           └───lib1.py
-    ├─── lib.py
-    ├─── test_lib.py
-    ├─── run.py
-    └─── provision.py
-```
-
-Each subdirectory in `bots` contains a bot. When writing bots, try to use the structure outlined
-above as an orientation.
-
-## Testing a bot's output
-
-If you just want to see how a bot reacts to a message, but don't want to set it up on a server,
-we have a little tool to help you out: `zulip-bot-shell`
-
-* [Install all requirements](#installing-a-development-version-of-the-zulip-bots-package).
-
-* Run `zulip-bot-shell` to test one of the bots in
-  [`zulip_bots/bots`](https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots).
-
-Example invocations are below:
-
-```
-> zulip-bot-shell converter
-
-Enter your message: "12 meter yard"
-Response: 12.0 meter = 13.12336 yard
-
-> zulip-bot-shell -b ~/followup.conf followup
-
-Enter your message: "Task completed"
-Response: stream: followup topic: foo_sender@zulip.com
-          from foo_sender@zulip.com: Task completed
-
-```
-
-Note that the `-b` (aka `--bot-config-file`) argument is for an optional third party
-config file (e.g., ~/giphy.conf), which only applies to certain types of bots.
-
-## Bot API
-
-This section documents functions available to the bot and the structure of the bot's config file.
-
-With this API, you *can*
-
-* intercept, view, and process messages sent by users on Zulip.
-* send out new messages as replies to the processed messages.
-
-With this API, you *cannot*
-
-* modify an intercepted message (you have to send a new message).
-* send messages on behalf of or impersonate other users.
-* intercept direct messages (except for direct messages with the bot as an
-explicit recipient).
-
-### usage
-
-*usage(self)*
-
-is called to retrieve information about the bot.
-
-#### Arguments
-
-* self - the instance the method is called on.
-
-#### Return values
-
-* A string describing the bot's functionality
-
-#### Example implementation
-
-```python
-def usage(self):
-    return '''
-        This plugin will allow users to flag messages
-        as being follow-up items.  Users should preface
-        messages with "@followup".
-        Before running this, make sure to create a channel
-        called "followup" that your API user can send to.
-        '''
-```
-
-### handle_message
-
-*handle_message(self, message, bot_handler)*
-
-handles user message.
-
-#### Arguments
-
-* self - the instance the method is called on.
-
-* message - a dictionary describing a Zulip message
-
-* bot_handler - used to interact with the server, e.g., to send a message
-
-#### Return values
-
-None.
-
-#### Example implementation
-
-```python
-  def handle_message(self, message, bot_handler):
-     original_content = message['content']
-     original_sender = message['sender_email']
-     new_content = original_content.replace('@followup',
-                                            'from %s:' % (original_sender,))
-
-     bot_handler.send_message(dict(
-         type='stream',
-         to='followup',
-         subject=message['sender_email'],
-         content=new_content,
-     ))
-```
-### bot_handler.send_message
-
-*bot_handler.send_message(message)*
-
-will send a message as the bot user.  Generally, this is less
-convenient than *send_reply*, but it offers additional flexibility
-about where the message is sent to.
-
-#### Arguments
-
-* message - a dictionary describing the message to be sent by the bot
-
-#### Example implementation
-
-```python
-bot_handler.send_message(dict(
-    type='stream', # can be 'stream' or 'private'
-    to=channel_name, # either the channel name or user's email
-    subject=subject, # message subject
-    content=message, # content of the sent message
-))
-```
-
-### bot_handler.send_reply
-
-*bot_handler.send_reply(message, response)*
-
-will reply to the triggering message to the same place the original
-message was sent to, with the content of the reply being *response*.
-
-#### Arguments
-
-* message - Dictionary containing information on message to respond to
- (provided by `handle_message`).
-* response - Response message from the bot (string).
-
-### bot_handler.update_message
-
-*bot_handler.update_message(message)*
-
-will edit the content of a previously sent message.
-
-#### Arguments
-
-* message - dictionary defining what message to edit and the new content
-
-#### Example
-
-From `zulip_bots/bots/incrementor/incrementor.py`:
-
-```python
-bot_handler.update_message(dict(
-    message_id=self.message_id, # id of message to be updated
-    content=str(self.number), # string with which to update message with
-))
-```
-
-### bot_handler.storage
-
-A common problem when writing an interactive bot is that you want to
-be able to store a bit of persistent state for the bot (e.g., for an
-RSVP bot, the RSVPs).  For a sufficiently complex bot, you want need
-your own database, but for simpler bots, we offer a convenient way for
-bot code to persistently store data.
-
-The interface for doing this is `bot_handler.storage`.
-
-The data is stored in the Zulip Server's database. Each bot user has
-an independent storage quota available to it.
-
-#### Performance considerations
-
-You can use `bot_handler.storage` in one of two ways:
-
-- **Direct access**: You can use bot_handler.storage directly, which
-will result in a round-trip to the server for each `get`, `put`, and
-`contains` call.
-- **Context manager**: Alternatively, you can use the `use_storage`
-context manager to minimize the number of round-trips to the server. We
-recommend writing bots with the context manager such that they
-automatically fetch data at the start of `handle_message` and submit the
-state to the server at the end.
-
-#### Context manager use_storage
-
-`use_storage(storage: BotStorage, keys: List[str])`
-
-The context manager fetches the data for the specified keys and stores
-them in a `CachedStorage` object with a `bot_handler.storage.get` call for
-each key, at the start. This object will not communicate with the server
-until manually calling flush or getting some values that are not previously
-fetched. After the context manager block is exited, it will automatically
-flush any changes made to the `CachedStorage` object to the server.
-
-##### Arguments
-* storage - a BotStorage object, i.e., `bot_handler.storage`
-* keys - a list of keys to fetch
-
-##### Example
-
-```python
-with use_storage(bot_handler.storage, ["foo", "bar"]) as cache:
-    print(cache.get("foo"))  # print the value of "foo"
-    cache.put("foo", "new value")  # update the value of "foo"
-# changes are automatically flushed to the server on exiting the block
-```
-
-#### bot_handler.storage methods
-
-When using the `use_storage` context manager, the `bot_handler.storage`
-methods on the yielded object will only operate on a cached version of the
-storage.
-
-#### bot_handler.storage.put
-
-*bot_handler.storage.put(key, value)*
-
-will store the value `value` in the entry `key`.
-
-##### Arguments
-
-* key - a UTF-8 string
-* value - a UTF-8 string
-
-##### Example
-
-```python
-bot_handler.storage.put("foo", "bar")  # set entry "foo" to "bar"
-```
-
-#### bot_handler.storage.get
-
-*bot_handler.storage.get(key)*
-
-will retrieve the value for the entry `key`.
-
-###### Arguments
-
-* key - a UTF-8 string
-
-##### Example
-
-```python
-bot_handler.storage.put("foo", "bar")
-print(bot_handler.storage.get("foo"))  # print "bar"
-```
-
-#### bot_handler.storage.contains
-
-*bot_handler.storage.contains(key)*
-
-will check if the entry `key` exists.
-
-Note that this will only check the cache, so it would return `False` if no
-previous call to `bot_handler.storage.get()` or `bot_handler.storage.put()`
-was made for `key`, since the bot was restarted.
-
-##### Arguments
-
-* key - a UTF-8 string
-
-##### Example
-
-```python
-bot_handler.storage.contains("foo")  # False
-bot_handler.storage.put("foo", "bar")
-bot_handler.storage.contains("foo")  # True
-```
-
-#### bot_handler.storage marshaling
-
-By default, `bot_handler.storage` accepts any object for keys and
-values, as long as it is JSON-able. Internally, the object then gets
-converted to an UTF-8 string. You can specify custom data marshaling
-by setting the functions `bot_handler.storage.marshal` and
-`bot_handler.storage.demarshal`. These functions parse your data on
-every call to `put` and `get`, respectively.
-
-#### Flushing cached data to the server
-
-When using the `use_storage` context manager, you can manually flush
-changes made to the cache to the server, using the below methods.
-
-#### cache.flush
-
-`cache.flush()`
-
-will flush all changes to the cache to the server.
-
-##### Example
-```python
-with use_storage(bot_handler.storage, ["foo", "bar"]) as cache:
-    cache.put("foo", "foo_value")  # update the value of "foo"
-    cache.put("bar", "bar_value")  # update the value of "bar"
-    cache.flush()  # manually flush both the changes to the server
-```
-
-#### cache.flush_one
-
-`cache.flush_one(key)`
-
-will flush the changes for the specified key to the server.
-
-##### Arguments
-
-- key - a UTF-8 string
-
-##### Example
-```python
-with use_storage(bot_handler.storage, ["foo", "bar"]) as cache:
-    cache.put("foo", "baz")  # update the value of "foo"
-    cache.put("bar", "bar_value")  # update the value of "bar"
-    cache.flush_one("foo")  # flush the changes to "foo" to the server
-```
-
-### Configuration file
-
-```
- [api]
- key=<api-key>
- email=<email>
- site=<dev-url>
-```
-
-* key - the API key you created for the bot; this is how Zulip knows
-  the request is from an authorized user.
-
-* email - the email address of the bot, e.g., `some-bot@zulip.com`
-
-* site - your development environment URL; if you are working on a
-  development environment hosted on your computer, use
-  `localhost:9991`
-
-## Writing tests for bots
-
-Bots, like most software that you want to work, should have unit tests. In this section,
-we detail our framework for writing unit tests for bots. We require that bots in the main
-[`python-zulip-api`](https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots)
-repository include a reasonable set of unit tests, so that future developers can easily
-refactor them.
-
-*Unit tests for bots make heavy use of mocking. If you want to get comfortable with mocking,
- mocking strategies, etc. you should check out our [mocking guide](
- https://zulip.readthedocs.io/en/latest/testing/testing-with-django.html#testing-with-mocks).*
-
-### A simple example
-
- Let's have a look at a simple test suite for the [`helloworld`](
- https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots/helloworld)
- bot.
-
-```python
-from zulip_bots.test_lib import StubBotTestCase
-
-class TestHelpBot(StubBotTestCase):
-    bot_name: str = "helloworld"
-
-    def test_bot(self) -> None:
-        dialog = [
-            ('', 'beep boop'),
-            ('help', 'beep boop'),
-            ('foo', 'beep boop'),
-        ]
-
-        self.verify_dialog(dialog)
-
-```
-
-The `helloworld` bot replies with "beep boop" to every message @-mentioning it.  We
-want our test to verify that the bot **actually** does that.
-
-Note that our helper method `verify_dialog` simulates the conversation for us, and
-we just need to set up a list of tuples with expected results.
-
-The best way to learn about bot tests is to read all the existing tests in the
-`bots` subdirectories.
-
-### Testing your test
-
-Once you have written a test suite, you want to verify that everything works as expected.
-
-* To test a bot in [Zulip's bot directory](
-  https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots):
-  `tools/test-bots <botname>`
-
-* To run all bot tests: `tools/test-bots`
-
-### Advanced testing
-
-This section shows advanced testing techniques for more complicated bots that have
-configuration files or interact with third-party APIs.
-*The code for the bot testing library can be found [here](
- https://github.com/zulip/python-zulip-api/blob/main/zulip_bots/zulip_bots/test_lib.py).*
-
-
-#### Testing bots with config files
-
-Some bots, such as [Giphy](
-https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots/giphy),
-support or require user configuration options to control how the bot works.
-
-To test such a bot, you can use the following pattern:
-
-    with self.mock_config_info(dict(api_key=12345)):
-        # self.verify_reply(...)
-
-`mock_config_info()` replaces the actual step of reading configuration from the file
-system and gives your test "dummy data" instead.
-
-#### Testing bots with internet access
-
-Some bots, such as [Giphy](
-https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots/giphy),
-depend on a third-party service, such as the Giphy web app, in order to work. Because
-we want our test suite to be reliable and not add load to these third-party APIs, tests
-for these services need to have "test fixtures": sample HTTP request/response pairs to
-be used by the tests. You can specify which one to use in your test code using the
-following helper method:
-
-    with self.mock_http_conversation('test_fixture_name'):
-        # self.assert_bot_response(...)
-
-`mock_http_conversation(fixture_name)` patches `requests.get` and returns the data specified
-in the file `fixtures/{fixture_name}.json`. Use the following JSON code as a skeleton for new
-fixtures:
-```json
-{
-  "request": {
-    "api_url": "http://api.example.com/",
-    "params": {
-    }
-  },
-  "response": {
-  },
-  "response-headers": {
-  }
-}
-```
-For an example, check out the [giphy bot](
-https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots/giphy).
-
-*Tip: You can use [requestbin](https://requestbin.com/) or a similar
-tool to capture payloads from the service your bot is interacting
-with.*
-
-#### Examples
-
-Check out our [bots](https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots)
-to see examples of bot tests.
-
-## Common problems
-
-* I modified my bot's code, yet the changes don't seem to have an effect.
-    * Ensure that you restarted the `zulip-run-bot` script.
-
-* My bot won't start
-    * Ensure that your API config file is correct (download the config file from the server).
-    * Ensure that you bot script is located in `zulip_bots/bots/<my-bot>/`
-    * Are you using your own Zulip development server? Ensure that you run your bot outside
-      the Vagrant environment.
-    * Some bots require Python 3. Try switching to a Python 3 environment before running
-      your bot.
-
-## Future direction
-
-The long-term plan for this bot system is to allow the same
-`ExternalBotHandler` code to eventually be usable in several contexts:
-
-* Run directly using the Zulip `call_on_each_message` API, which is
-  how the implementation above works.  This is great for quick
-  development with minimal setup.
-* Run in a simple Python web server, processing messages
-  received from Zulip's outgoing webhooks integration.
-* For bots merged into the mainline Zulip codebase, enabled via a
-  button in the Zulip web UI, with no code deployment effort required.
-
-## Related articles
-
-* [Non-webhook integrations](/api/non-webhook-integrations)
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Configuring the Python bindings](/api/configuring-python-bindings)

babel.config.js

@@ -0,0 +1,26 @@
+"use strict";
+
+module.exports = {
+    plugins: [
+        [
+            "formatjs",
+            {
+                additionalFunctionNames: ["$t", "$t_html"],
+                overrideIdFn: (id, defaultMessage) => defaultMessage,
+            },
+        ],
+    ],
+    presets: [
+        [
+            "@babel/preset-env",
+            {
+                corejs: "3.6",
+                loose: true, // Loose mode for…of loops are 5× faster in Firefox
+                shippedProposals: true,
+                useBuiltIns: "usage",
+            },
+        ],
+        "@babel/typescript",
+    ],
+    sourceType: "unambiguous",
+};

confirmation/migrations/0001_initial.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("contenttypes", "0001_initial"),
     ]

confirmation/migrations/0001_squashed_0014_confirmation_confirmatio_content_80155a_idx.py

@@ -1,90 +0,0 @@
-# Generated by Django 5.0.7 on 2024-08-13 19:41
-
-import django.db.models.deletion
-import django.utils.timezone
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    replaces = [
-        ("confirmation", "0001_initial"),
-        ("confirmation", "0002_realmcreationkey"),
-        ("confirmation", "0003_emailchangeconfirmation"),
-        ("confirmation", "0004_remove_confirmationmanager"),
-        ("confirmation", "0005_confirmation_realm"),
-        ("confirmation", "0006_realmcreationkey_presume_email_valid"),
-        ("confirmation", "0007_add_indexes"),
-        ("confirmation", "0008_confirmation_expiry_date"),
-        ("confirmation", "0009_confirmation_expiry_date_backfill"),
-        ("confirmation", "0010_alter_confirmation_expiry_date"),
-        ("confirmation", "0011_alter_confirmation_expiry_date"),
-        ("confirmation", "0012_alter_confirmation_id"),
-        ("confirmation", "0013_alter_realmcreationkey_id"),
-        ("confirmation", "0014_confirmation_confirmatio_content_80155a_idx"),
-    ]
-
-    initial = True
-
-    dependencies = [
-        ("contenttypes", "0001_initial"),
-        ("zerver", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="RealmCreationKey",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                (
-                    "creation_key",
-                    models.CharField(db_index=True, max_length=40, verbose_name="activation key"),
-                ),
-                (
-                    "date_created",
-                    models.DateTimeField(default=django.utils.timezone.now, verbose_name="created"),
-                ),
-                ("presume_email_valid", models.BooleanField(default=False)),
-            ],
-        ),
-        migrations.CreateModel(
-            name="Confirmation",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                ("object_id", models.PositiveIntegerField(db_index=True)),
-                ("date_sent", models.DateTimeField(db_index=True)),
-                ("confirmation_key", models.CharField(db_index=True, max_length=40)),
-                (
-                    "content_type",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype"
-                    ),
-                ),
-                ("type", models.PositiveSmallIntegerField()),
-                (
-                    "realm",
-                    models.ForeignKey(
-                        null=True, on_delete=django.db.models.deletion.CASCADE, to="zerver.realm"
-                    ),
-                ),
-                ("expiry_date", models.DateTimeField(db_index=True, null=True)),
-            ],
-            options={
-                "unique_together": {("type", "confirmation_key")},
-                "indexes": [
-                    models.Index(
-                        fields=["content_type", "object_id"], name="confirmatio_content_80155a_idx"
-                    )
-                ],
-            },
-        ),
-    ]

confirmation/migrations/0002_realmcreationkey.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0001_initial"),
     ]

confirmation/migrations/0003_emailchangeconfirmation.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0002_realmcreationkey"),
     ]

confirmation/migrations/0004_remove_confirmationmanager.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0003_emailchangeconfirmation"),
     ]

confirmation/migrations/0005_confirmation_realm.py

@@ -4,8 +4,9 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("zerver", "0001_initial"),
+        ("zerver", "0124_stream_enable_notifications"),
         ("confirmation", "0004_remove_confirmationmanager"),
     ]
 

confirmation/migrations/0006_realmcreationkey_presume_email_valid.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0005_confirmation_realm"),
     ]