compose fade: Add test for want_normal_display.

.codespellignore

@@ -25,6 +25,3 @@ forin
 uper
 slac
 couldn
-ges
-assertIn
-thirdparty

.editorconfig

@@ -8,9 +8,8 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[[shell]]
-binary_next_line = true
-switch_case_indent = true
+binary_next_line = true  # for shfmt
+switch_case_indent = true  # for shfmt
 
 [{*.{js,json,ts},check-openapi}]
 max_line_length = 100

.eslintrc.js

@@ -1,293 +0,0 @@
-"use strict";
-
-const confusingBrowserGlobals = require("confusing-browser-globals");
-
-module.exports = {
-    root: true,
-    env: {
-        es2020: true,
-        node: true,
-    },
-    extends: [
-        "eslint:recommended",
-        "plugin:import/errors",
-        "plugin:import/warnings",
-        "plugin:no-jquery/recommended",
-        "plugin:no-jquery/deprecated",
-        "plugin:unicorn/recommended",
-        "prettier",
-    ],
-    parser: "@babel/eslint-parser",
-    parserOptions: {
-        requireConfigFile: false,
-        warnOnUnsupportedTypeScriptVersion: false,
-        sourceType: "unambiguous",
-    },
-    plugins: ["formatjs", "no-jquery"],
-    settings: {
-        formatjs: {
-            additionalFunctionNames: ["$t", "$t_html"],
-        },
-        "no-jquery": {
-            collectionReturningPlugins: {
-                expectOne: "always",
-            },
-            variablePattern: "^\\$(?!t$|t_html$).",
-        },
-    },
-    reportUnusedDisableDirectives: true,
-    rules: {
-        "array-callback-return": "error",
-        "arrow-body-style": "error",
-        "block-scoped-var": "error",
-        "consistent-return": "error",
-        curly: "error",
-        "dot-notation": "error",
-        eqeqeq: "error",
-        "formatjs/enforce-default-message": ["error", "literal"],
-        "formatjs/enforce-placeholders": [
-            "error",
-            {ignoreList: ["b", "code", "em", "i", "kbd", "p", "strong"]},
-        ],
-        "formatjs/no-id": "error",
-        "guard-for-in": "error",
-        "import/extensions": "error",
-        "import/first": "error",
-        "import/newline-after-import": "error",
-        "import/no-cycle": ["error", {ignoreExternal: true}],
-        "import/no-duplicates": "error",
-        "import/no-self-import": "error",
-        "import/no-unresolved": "off",
-        "import/no-useless-path-segments": "error",
-        "import/order": ["error", {alphabetize: {order: "asc"}, "newlines-between": "always"}],
-        "import/unambiguous": "error",
-        "lines-around-directive": "error",
-        "new-cap": "error",
-        "no-alert": "error",
-        "no-array-constructor": "error",
-        "no-bitwise": "error",
-        "no-caller": "error",
-        "no-catch-shadow": "error",
-        "no-constant-condition": ["error", {checkLoops: false}],
-        "no-div-regex": "error",
-        "no-else-return": "error",
-        "no-eq-null": "error",
-        "no-eval": "error",
-        "no-implicit-coercion": "error",
-        "no-implied-eval": "error",
-        "no-inner-declarations": "off",
-        "no-iterator": "error",
-        "no-jquery/no-append-html": "error",
-        "no-jquery/no-constructor-attributes": "error",
-        "no-jquery/no-parse-html-literal": "error",
-        "no-label-var": "error",
-        "no-labels": "error",
-        "no-loop-func": "error",
-        "no-multi-str": "error",
-        "no-native-reassign": "error",
-        "no-new-func": "error",
-        "no-new-object": "error",
-        "no-new-wrappers": "error",
-        "no-octal-escape": "error",
-        "no-plusplus": "error",
-        "no-proto": "error",
-        "no-restricted-globals": ["error", ...confusingBrowserGlobals],
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-self-compare": "error",
-        "no-sync": "error",
-        "no-throw-literal": "error",
-        "no-undef-init": "error",
-        "no-unneeded-ternary": ["error", {defaultAssignment: false}],
-        "no-unused-expressions": "error",
-        "no-unused-vars": [
-            "error",
-            {args: "all", argsIgnorePattern: "^_", ignoreRestSiblings: true},
-        ],
-        "no-use-before-define": ["error", {functions: false}],
-        "no-useless-concat": "error",
-        "no-useless-constructor": "error",
-        "no-var": "error",
-        "object-shorthand": ["error", "always", {avoidExplicitReturnArrows: true}],
-        "one-var": ["error", "never"],
-        "prefer-arrow-callback": "error",
-        "prefer-const": ["error", {ignoreReadBeforeAssign: true}],
-        radix: "error",
-        "sort-imports": ["error", {ignoreDeclarationSort: true}],
-        "spaced-comment": ["error", "always", {markers: ["/"]}],
-        strict: "error",
-        "unicorn/consistent-function-scoping": "off",
-        "unicorn/explicit-length-check": "off",
-        "unicorn/filename-case": "off",
-        "unicorn/no-await-expression-member": "off",
-        "unicorn/no-negated-condition": "off",
-        "unicorn/no-null": "off",
-        "unicorn/no-process-exit": "off",
-        "unicorn/no-useless-undefined": "off",
-        "unicorn/numeric-separators-style": "off",
-        "unicorn/prefer-module": "off",
-        "unicorn/prefer-node-protocol": "off",
-        "unicorn/prefer-string-raw": "off",
-        "unicorn/prefer-ternary": "off",
-        "unicorn/prefer-top-level-await": "off",
-        "unicorn/prevent-abbreviations": "off",
-        "unicorn/switch-case-braces": "off",
-        "valid-typeof": ["error", {requireStringLiterals: true}],
-        yoda: "error",
-    },
-    overrides: [
-        {
-            files: ["web/tests/**"],
-            rules: {
-                "no-jquery/no-selector-prop": "off",
-            },
-        },
-        {
-            files: ["web/e2e-tests/**"],
-            globals: {
-                zulip_test: false,
-            },
-        },
-        {
-            files: ["web/src/**"],
-            globals: {
-                StripeCheckout: false,
-            },
-        },
-        {
-            files: ["**/*.ts"],
-            extends: [
-                "plugin:@typescript-eslint/strict-type-checked",
-                "plugin:@typescript-eslint/stylistic-type-checked",
-                "plugin:import/typescript",
-            ],
-            parserOptions: {
-                project: "tsconfig.json",
-            },
-            settings: {
-                "import/resolver": {
-                    node: {
-                        extensions: [".ts", ".d.ts", ".js"], // https://github.com/import-js/eslint-plugin-import/issues/2267
-                    },
-                },
-            },
-            globals: {
-                JQuery: false,
-            },
-            rules: {
-                // Disable base rule to avoid conflict
-                "no-use-before-define": "off",
-
-                "@typescript-eslint/consistent-type-assertions": [
-                    "error",
-                    {assertionStyle: "never"},
-                ],
-                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
-                "@typescript-eslint/consistent-type-imports": "error",
-                "@typescript-eslint/explicit-function-return-type": [
-                    "error",
-                    {allowExpressions: true},
-                ],
-                "@typescript-eslint/member-ordering": "error",
-                "@typescript-eslint/method-signature-style": "error",
-                "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-unnecessary-condition": "off",
-                "@typescript-eslint/no-unnecessary-qualifier": "error",
-                "@typescript-eslint/no-unused-vars": [
-                    "error",
-                    {args: "all", argsIgnorePattern: "^_", ignoreRestSiblings: true},
-                ],
-                "@typescript-eslint/no-use-before-define": ["error", {functions: false}],
-                "@typescript-eslint/parameter-properties": "error",
-                "@typescript-eslint/promise-function-async": "error",
-                "@typescript-eslint/restrict-plus-operands": ["error", {}],
-                "@typescript-eslint/restrict-template-expressions": ["error", {}],
-                "no-undef": "error",
-            },
-        },
-        {
-            files: ["**/*.d.ts"],
-            rules: {
-                "import/unambiguous": "off",
-            },
-        },
-        {
-            files: ["web/e2e-tests/**", "web/tests/**"],
-            globals: {
-                CSS: false,
-                document: false,
-                navigator: false,
-                window: false,
-            },
-            rules: {
-                "formatjs/no-id": "off",
-                "new-cap": "off",
-                "no-sync": "off",
-                "unicorn/prefer-prototype-methods": "off",
-            },
-        },
-        {
-            files: ["web/debug-require.js"],
-            env: {
-                browser: true,
-                es2020: false,
-            },
-            rules: {
-                // Don’t require ES features that PhantomJS doesn’t support
-                // TODO: Toggle these settings now that we don't use PhantomJS
-                "no-var": "off",
-                "object-shorthand": "off",
-                "prefer-arrow-callback": "off",
-            },
-        },
-        {
-            files: ["web/shared/**", "web/src/**", "web/third/**"],
-            env: {
-                browser: true,
-                node: false,
-            },
-            globals: {
-                DEVELOPMENT: false,
-                ZULIP_VERSION: false,
-            },
-            rules: {
-                "no-console": "error",
-            },
-            settings: {
-                "import/resolver": {
-                    webpack: {
-                        config: "./web/webpack.config.ts",
-                    },
-                },
-            },
-        },
-        {
-            files: ["web/shared/**"],
-            env: {
-                browser: false,
-                "shared-node-browser": true,
-            },
-            rules: {
-                "import/no-restricted-paths": [
-                    "error",
-                    {
-                        zones: [
-                            {
-                                target: "./web/shared",
-                                from: ".",
-                                except: ["./node_modules", "./web/shared"],
-                            },
-                        ],
-                    },
-                ],
-                "unicorn/prefer-string-replace-all": "off",
-            },
-        },
-        {
-            files: ["web/server/**"],
-            env: {
-                node: true,
-            },
-        },
-    ],
-};

.eslintrc.json

@@ -0,0 +1,282 @@
+{
+    "root": true,
+    "env": {
+        "es2020": true,
+        "node": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:import/errors",
+        "plugin:import/warnings",
+        "plugin:no-jquery/recommended",
+        "plugin:no-jquery/deprecated",
+        "plugin:unicorn/recommended",
+        "prettier"
+    ],
+    "parser": "@babel/eslint-parser",
+    "parserOptions": {
+        "requireConfigFile": false,
+        "warnOnUnsupportedTypeScriptVersion": false,
+        "sourceType": "unambiguous"
+    },
+    "plugins": ["formatjs", "no-jquery"],
+    "settings": {
+        "formatjs": {
+            "additionalFunctionNames": ["$t", "$t_html"]
+        },
+        "no-jquery": {
+            "collectionReturningPlugins": {
+                "expectOne": "always"
+            },
+            "variablePattern": "^\\$(?!t$|t_html$)."
+        }
+    },
+    "reportUnusedDisableDirectives": true,
+    "rules": {
+        "array-callback-return": "error",
+        "arrow-body-style": "error",
+        "block-scoped-var": "error",
+        "consistent-return": "error",
+        "curly": "error",
+        "dot-notation": "error",
+        "eqeqeq": "error",
+        "formatjs/enforce-default-message": ["error", "literal"],
+        "formatjs/enforce-placeholders": [
+            "error",
+            {"ignoreList": ["b", "code", "em", "i", "kbd", "p", "strong"]}
+        ],
+        "formatjs/no-id": "error",
+        "guard-for-in": "error",
+        "import/extensions": "error",
+        "import/first": "error",
+        "import/newline-after-import": "error",
+        "import/no-cycle": ["error", {"ignoreExternal": true}],
+        "import/no-duplicates": "error",
+        "import/no-self-import": "error",
+        "import/no-unresolved": "off",
+        "import/no-useless-path-segments": "error",
+        "import/order": ["error", {"alphabetize": {"order": "asc"}, "newlines-between": "always"}],
+        "import/unambiguous": "error",
+        "lines-around-directive": "error",
+        "new-cap": "error",
+        "no-alert": "error",
+        "no-array-constructor": "error",
+        "no-bitwise": "error",
+        "no-caller": "error",
+        "no-catch-shadow": "error",
+        "no-constant-condition": ["error", {"checkLoops": false}],
+        "no-div-regex": "error",
+        "no-else-return": "error",
+        "no-eq-null": "error",
+        "no-eval": "error",
+        "no-implicit-coercion": "error",
+        "no-implied-eval": "error",
+        "no-inner-declarations": "off",
+        "no-iterator": "error",
+        "no-jquery/no-constructor-attributes": "error",
+        "no-jquery/no-parse-html-literal": "error",
+        "no-label-var": "error",
+        "no-labels": "error",
+        "no-loop-func": "error",
+        "no-multi-str": "error",
+        "no-native-reassign": "error",
+        "no-new-func": "error",
+        "no-new-object": "error",
+        "no-new-wrappers": "error",
+        "no-octal-escape": "error",
+        "no-plusplus": "error",
+        "no-proto": "error",
+        "no-return-assign": "error",
+        "no-script-url": "error",
+        "no-self-compare": "error",
+        "no-sync": "error",
+        "no-throw-literal": "error",
+        "no-undef-init": "error",
+        "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
+        "no-unused-expressions": "error",
+        "no-unused-vars": [
+            "error",
+            {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
+        ],
+        "no-use-before-define": ["error", {"functions": false}],
+        "no-useless-concat": "error",
+        "no-useless-constructor": "error",
+        "no-var": "error",
+        "object-shorthand": ["error", "always", {"avoidExplicitReturnArrows": true}],
+        "one-var": ["error", "never"],
+        "prefer-arrow-callback": "error",
+        "prefer-const": ["error", {"ignoreReadBeforeAssign": true}],
+        "radix": "error",
+        "sort-imports": ["error", {"ignoreDeclarationSort": true}],
+        "spaced-comment": ["error", "always", {"markers": ["/"]}],
+        "strict": "error",
+        "unicorn/consistent-function-scoping": "off",
+        "unicorn/explicit-length-check": "off",
+        "unicorn/filename-case": "off",
+        "unicorn/no-await-expression-member": "off",
+        "unicorn/no-negated-condition": "off",
+        "unicorn/no-null": "off",
+        "unicorn/no-process-exit": "off",
+        "unicorn/no-useless-undefined": "off",
+        "unicorn/numeric-separators-style": "off",
+        "unicorn/prefer-module": "off",
+        "unicorn/prefer-node-protocol": "off",
+        "unicorn/prefer-ternary": "off",
+        "unicorn/prefer-top-level-await": "off",
+        "unicorn/prevent-abbreviations": "off",
+        "unicorn/switch-case-braces": "off",
+        "valid-typeof": ["error", {"requireStringLiterals": true}],
+        "yoda": "error"
+    },
+    "overrides": [
+        {
+            "files": ["web/tests/**"],
+            "rules": {
+                "no-jquery/no-selector-prop": "off"
+            }
+        },
+        {
+            "files": ["web/e2e-tests/**"],
+            "globals": {
+                "zulip_test": false
+            }
+        },
+        {
+            "files": ["web/src/**"],
+            "globals": {
+                "StripeCheckout": false
+            }
+        },
+        {
+            "files": ["**/*.ts"],
+            "extends": [
+                "plugin:@typescript-eslint/recommended",
+                "plugin:@typescript-eslint/recommended-requiring-type-checking",
+                "plugin:@typescript-eslint/strict",
+                "plugin:import/typescript"
+            ],
+            "parserOptions": {
+                "project": "tsconfig.json"
+            },
+            "settings": {
+                "import/resolver": {
+                    "node": {
+                        "extensions": [".ts", ".d.ts", ".js"] // https://github.com/import-js/eslint-plugin-import/issues/2267
+                    }
+                }
+            },
+            "globals": {
+                "JQuery": false
+            },
+            "rules": {
+                // Disable base rule to avoid conflict
+                "no-use-before-define": "off",
+
+                "@typescript-eslint/consistent-type-assertions": [
+                    "error",
+                    {"assertionStyle": "never"}
+                ],
+                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
+                "@typescript-eslint/consistent-type-imports": "error",
+                "@typescript-eslint/explicit-function-return-type": [
+                    "error",
+                    {"allowExpressions": true}
+                ],
+                "@typescript-eslint/member-ordering": "error",
+                "@typescript-eslint/no-non-null-assertion": "off",
+                "@typescript-eslint/no-unnecessary-condition": "off",
+                "@typescript-eslint/no-unnecessary-qualifier": "error",
+                "@typescript-eslint/no-unsafe-argument": "off",
+                "@typescript-eslint/no-unsafe-assignment": "off",
+                "@typescript-eslint/no-unsafe-call": "off",
+                "@typescript-eslint/no-unsafe-member-access": "off",
+                "@typescript-eslint/no-unsafe-return": "off",
+                "@typescript-eslint/no-unused-vars": [
+                    "error",
+                    {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
+                ],
+                "@typescript-eslint/no-use-before-define": ["error", {"functions": false}],
+                "@typescript-eslint/parameter-properties": "error",
+                "@typescript-eslint/promise-function-async": "error",
+                "no-undef": "error"
+            }
+        },
+        {
+            "files": ["**/*.d.ts"],
+            "rules": {
+                "import/unambiguous": "off"
+            }
+        },
+        {
+            "files": ["web/e2e-tests/**", "web/tests/**"],
+            "globals": {
+                "CSS": false,
+                "document": false,
+                "navigator": false,
+                "window": false
+            },
+            "rules": {
+                "formatjs/no-id": "off",
+                "new-cap": "off",
+                "no-sync": "off",
+                "unicorn/prefer-prototype-methods": "off"
+            }
+        },
+        {
+            "files": ["web/debug-require.js"],
+            "env": {
+                "browser": true,
+                "es2020": false
+            },
+            "rules": {
+                // Don’t require ES features that PhantomJS doesn’t support
+                // TODO: Toggle these settings now that we don't use PhantomJS
+                "no-var": "off",
+                "object-shorthand": "off",
+                "prefer-arrow-callback": "off"
+            }
+        },
+        {
+            "files": ["web/shared/**", "web/src/**", "web/third/**"],
+            "env": {
+                "browser": true,
+                "node": false
+            },
+            "globals": {
+                "ZULIP_VERSION": false
+            },
+            "rules": {
+                "no-console": "error"
+            },
+            "settings": {
+                "import/resolver": {
+                    "webpack": {
+                        "config": "./web/webpack.config.ts"
+                    }
+                }
+            }
+        },
+        {
+            "files": ["web/shared/**"],
+            "env": {
+                "browser": false,
+                "shared-node-browser": true
+            },
+            "rules": {
+                "import/no-restricted-paths": [
+                    "error",
+                    {
+                        "zones": [
+                            {
+                                "target": "./web/shared",
+                                "from": ".",
+                                "except": ["./node_modules", "./web/shared"]
+                            }
+                        ]
+                    }
+                ],
+                "unicorn/prefer-string-replace-all": "off"
+            }
+        }
+    ]
+}

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -11,7 +11,6 @@ labels: ["bug"]
 **Zulip Server and web app version:**
 
 - [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 8.0+
 - [ ] Zulip Server 7.0+
 - [ ] Zulip Server 6.0+
 - [ ] Zulip Server 5.0 or older

.github/workflows/codeql-analysis.yml

@@ -26,15 +26,15 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v2
 
         # Override language selection by uncommenting this and choosing your languages
         # with:
         #   languages: go, javascript, csharp, python, cpp, java
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v2

.github/workflows/production-suite.yml

@@ -39,13 +39,13 @@ jobs:
   production_build:
     # This job builds a release tarball from the current commit, which
     # will be used for all of the following install/upgrade tests.
-    name: Ubuntu 22.04 production build
+    name: Ubuntu 20.04 production build
     runs-on: ubuntu-latest
 
     # Docker images are built from 'tools/ci/Dockerfile'; the comments at
     # the top explain how to build and upload these images.
-    # Ubuntu 22.04 ships with Python 3.10.12.
-    container: zulip/ci:jammy
+    # Ubuntu 20.04 ships with Python 3.8.10.
+    container: zulip/ci:focal
 
     steps:
       - name: Add required permissions
@@ -64,7 +64,7 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: Create cache directories
         run: |
@@ -73,30 +73,30 @@ jobs:
           sudo chown -R github "${dirs[@]}"
 
       - name: Restore pnpm store
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /__w/.pnpm-store
-          key: v1-pnpm-store-jammy-${{ hashFiles('pnpm-lock.yaml') }}
+          key: v1-pnpm-store-focal-${{ hashFiles('pnpm-lock.yaml') }}
 
       - name: Restore python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /srv/zulip-venv-cache
-          key: v1-venv-jammy-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-jammy
+          key: v1-venv-focal-${{ hashFiles('requirements/dev.txt') }}
+          restore-keys: v1-venv-focal
 
       - name: Restore emoji cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /srv/zulip-emoji-cache
-          key: v1-emoji-jammy-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
-          restore-keys: v1-emoji-jammy
+          key: v1-emoji-focal-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
+          restore-keys: v1-emoji-focal
 
       - name: Build production tarball
         run: ./tools/ci/production-build
 
       - name: Upload production build artifacts for install jobs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: production-tarball
           path: /tmp/production-build
@@ -135,20 +135,25 @@ jobs:
         include:
           # Docker images are built from 'tools/ci/Dockerfile'; the comments at
           # the top explain how to build and upload these images.
+          - docker_image: zulip/ci:focal
+            name: Ubuntu 20.04 production install and PostgreSQL upgrade with pgroonga
+            os: focal
+            extra-args: ""
+
           - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 production install and PostgreSQL upgrade with pgroonga
+            name: Ubuntu 22.04 production install
             os: jammy
             extra-args: ""
 
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 production install
-            os: noble
-            extra-args: ""
+          - docker_image: zulip/ci:bullseye
+            name: Debian 11 production install with custom db name and user
+            os: bullseye
+            extra-args: --test-custom-db
 
           - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install with custom db name and user
+            name: Debian 12 production install
             os: bookworm
-            extra-args: --test-custom-db
+            extra-args: ""
 
     name: ${{ matrix.name  }}
     container:
@@ -159,7 +164,7 @@ jobs:
 
     steps:
       - name: Download built production tarball
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: production-tarball
           path: /tmp
@@ -171,7 +176,7 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-          # Since actions/download-artifact@v4 loses all the permissions
+          # Since actions/download-artifact@v2 loses all the permissions
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade-pg
           chmod +x /tmp/production-pgroonga
@@ -192,19 +197,19 @@ jobs:
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
 
       - name: Install pgroonga
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-pgroonga
 
       - name: Verify install after installing pgroonga
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
 
       - name: Upgrade postgresql
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-upgrade-pg
 
       - name: Verify install after upgrading postgresql
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
 
       - name: Generate failure report string
@@ -237,15 +242,18 @@ jobs:
         include:
           # Docker images are built from 'tools/ci/Dockerfile.prod'; the comments at
           # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:jammy-6.0
+          - docker_image: zulip/ci:focal-3.2
+            name: 3.2 Version Upgrade
+            os: focal
+          - docker_image: zulip/ci:bullseye-4.2
+            name: 4.2 Version Upgrade
+            os: bullseye
+          - docker_image: zulip/ci:bullseye-5.0
+            name: 5.0 Version Upgrade
+            os: bullseye
+          - docker_image: zulip/ci:bullseye-6.0
             name: 6.0 Version Upgrade
-            os: jammy
-          - docker_image: zulip/ci:bookworm-7.0
-            name: 7.0 Version Upgrade
-            os: bookworm
-          - docker_image: zulip/ci:bookworm-8.0
-            name: 8.0 Version Upgrade
-            os: bookworm
+            os: bullseye
 
     name: ${{ matrix.name  }}
     container:
@@ -256,7 +264,7 @@ jobs:
 
     steps:
       - name: Download built production tarball
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
           name: production-tarball
           path: /tmp
@@ -268,7 +276,7 @@ jobs:
           # cache action to work. It is owned by root currently.
           sudo chmod -R 0777 /__w/_temp/
 
-          # Since actions/download-artifact@v4 loses all the permissions
+          # Since actions/download-artifact@v2 loses all the permissions
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade
           chmod +x /tmp/production-verify
@@ -280,6 +288,17 @@ jobs:
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
+      - name: Temporarily bootstrap PostgreSQL upgrades
+        # https://chat.zulip.org/#narrow/stream/43-automated-testing/topic/postgres.20client.20upgrade.20failures/near/1640444
+        # On Debian, there is an ordering issue with post-install maintainer
+        # scripts when postgresql-client-common is upgraded at the same time as
+        # postgresql-client and postgresql-client-15.  Upgrade just
+        # postgresql-client-common first, so the main upgrade process can
+        # succeed.  This is a _temporary_ work-around to improve CI signal, as
+        # the failure does represent a real failure that production systems may
+        # encounter.
+        run: sudo apt-get update && sudo apt-get install -y --only-upgrade postgresql-client-common
+
       - name: Upgrade production
         run: sudo /tmp/production-upgrade
 

.github/workflows/update-oneclick-apps.yml

@@ -9,7 +9,7 @@ jobs:
   update-digitalocean-oneclick-app:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Update DigitalOcean one click app
         env:
           DIGITALOCEAN_API_KEY: ${{ secrets.ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY }}

.github/workflows/zulip-ci.yml

@@ -30,22 +30,28 @@ jobs:
         include:
           # Base images are built using `tools/ci/Dockerfile.prod.template`.
           # The comments at the top explain how to build and upload these images.
-          # Ubuntu 22.04 ships with Python 3.10.12.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 (Python 3.10, backend + frontend)
-            os: jammy
+          # Ubuntu 20.04 ships with Python 3.8.10.
+          - docker_image: zulip/ci:focal
+            name: Ubuntu 20.04 (Python 3.8, backend + frontend)
+            os: focal
             include_documentation_tests: false
             include_frontend_tests: true
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend + documentation)
-            os: bookworm
+          # Debian 11 ships with Python 3.9.2.
+          - docker_image: zulip/ci:bullseye
+            name: Debian 11 (Python 3.9, backend + documentation)
+            os: bullseye
             include_documentation_tests: true
             include_frontend_tests: false
-          # Ubuntu 24.04 ships with Python 3.12.2.
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 (Python 3.12, backend)
-            os: noble
+          # Ubuntu 22.04 ships with Python 3.10.4.
+          - docker_image: zulip/ci:jammy
+            name: Ubuntu 22.04 (Python 3.10, backend)
+            os: jammy
+            include_documentation_tests: false
+            include_frontend_tests: false
+          # Debian 12 ships with Python 3.11.2.
+          - docker_image: zulip/ci:bookworm
+            name: Debian 12 (Python 3.11, backend)
+            os: bookworm
             include_documentation_tests: false
             include_frontend_tests: false
 
@@ -62,7 +68,7 @@ jobs:
       HOME: /home/github/
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: Create cache directories
         run: |
@@ -71,20 +77,20 @@ jobs:
           sudo chown -R github "${dirs[@]}"
 
       - name: Restore pnpm store
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /__w/.pnpm-store
           key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
 
       - name: Restore python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /srv/zulip-venv-cache
           key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/dev.txt') }}
           restore-keys: v1-venv-${{ matrix.os }}
 
       - name: Restore emoji cache
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: /srv/zulip-emoji-cache
           key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json', 'tools/setup/emoji/build_emoji', 'tools/setup/emoji/emoji_setup_utils.py', 'tools/setup/emoji/emoji_names.py', 'package.json') }}
@@ -167,7 +173,7 @@ jobs:
       - name: Run backend tests
         run: |
           source tools/ci/activate-venv
-          ./tools/test-backend ${{ matrix.os != 'bookworm' && '--coverage' || '' }} --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
+          ./tools/test-backend --coverage --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
 
       - name: Run mypy
         run: |
@@ -215,7 +221,7 @@ jobs:
           fi
 
       - name: Test locked requirements
-        if: ${{ matrix.os == 'jammy' }}
+        if: ${{ matrix.os == 'focal' }}
         run: |
           . /srv/zulip-py3-venv/bin/activate && \
           ./tools/test-locked-requirements
@@ -225,15 +231,14 @@ jobs:
         # Only upload coverage when both frontend and backend
         # tests are run.
         if: ${{ matrix.include_frontend_tests }}
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v3
         with:
           files: var/coverage.xml,var/node-coverage/lcov.info
-          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Store Puppeteer artifacts
         # Upload these on failure, as well
         if: ${{ always() && matrix.include_frontend_tests }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: puppeteer
           path: ./var/puppeteer

.gitignore

@@ -17,17 +17,13 @@
 # See `git help ignore` for details on the format.
 
 ## Config files for the dev environment
-/zproject/apns-dev.pem
-/zproject/apns-dev-key.p8
 /zproject/dev-secrets.conf
-/zproject/custom_dev_settings.py
 /tools/conf.ini
 /tools/custom_provision
 /tools/droplets/conf.ini
 
 ## Byproducts of setting up and using the dev environment
 *.pyc
-*.tsbuildinfo
 package-lock.json
 
 /.vagrant

.mailmap

@@ -18,9 +18,6 @@ acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
 Adam Benesh <Adam.Benesh@gmail.com>
 Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
 Adarsh Tiwari <xoldyckk@gmail.com>
-Aditya Chaudhary <aditya.chaudhary1558@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com> <78212328+adnan-td@users.noreply.github.com>
 Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
 Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
@@ -32,26 +29,18 @@ Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
 Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
 Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
 aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Aryan Bhokare <aryan1bhokare@gmail.com>
-Aryan Bhokare <aryan1bhokare@gmail.com> <92683836+aryan-bhokare@users.noreply.github.com>
 Aryan Shridhar <aryanshridhar7@gmail.com>
 Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
 Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
 Austin Riba <austin@zulip.com> <austin@m51.io>
-Bedo Khaled <bedokhaled66@gmail.com>
-Bedo Khaled <bedokhaled66@gmail.com> <64221784+abdelrahman725@users.noreply.github.com>
 BIKI DAS <bikid475@gmail.com>
 Brijmohan Siyag <brijsiyag@gmail.com>
-Brock Whittaker <whittakerbrock@gmail.com> <bjwhitta@asu.edu>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.com>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulipchat.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brockwhittaker@Brocks-MacBook.local>
+Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
+Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
+Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
 Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
 Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-codewithnick <nikhilsingh526452@gmail.com>
 Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dhruv Goyal <dhruvgoyal.dev@gmail.com>
 Dinesh <chdinesh1089@gmail.com>
 Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
 Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
@@ -71,8 +60,6 @@ Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-John Lu <JohnLu10212004@gmail.com>
-John Lu <JohnLu10212004@gmail.com> <87673068+JohnLu2004@users.noreply.github.com>
 Joseph Ho <josephho678@gmail.com>
 Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
 Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
@@ -80,10 +67,7 @@ Karl Stolley <karl@zulip.com> <karl@stolley.dev>
 Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
 Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
 Kevin Scott <kevin.scott.98@gmail.com>
-Kislay Verma <kislayuv27@gmail.com>
-Kunal Sharma <v.shm.kunal@gmail.com>
 Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com> <lalits01@smartek21.com>
 Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
 Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
 m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
@@ -91,17 +75,12 @@ m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.
 Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
 Matt Keller <matt@zulip.com>
 Matt Keller <matt@zulip.com> <m@cognusion.com>
-Nehal Sharma <bablinaneh@gmail.com>
-Nehal Sharma <bablinaneh@gmail.com> <68962290+N-Shar-ma@users.noreply.github.com>
-Nimish Medatwal <medatwalnimish@gmail.com>
 Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
 nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
 Palash Baderia <palash.baderia@outlook.com>
 Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
 Palash Raghuwanshi <singhpalash0@gmail.com>
 Parth <mittalparth22@gmail.com>
-Pratik Chanda <pratikchanda2000@gmail.com>
-Pratik Solanki <pratiksolanki2021@gmail.com>
 Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
 Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
 Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
@@ -113,19 +92,13 @@ Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
 Rixant Rokaha <rixantrokaha@gmail.com>
 Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
 Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Rohan Gudimetla <rohan.gudimetla07@gmail.com>
 Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
 Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Sanchit Sharma <ssharmas10662@gmail.com>
 Satyam Bansal <sbansal1999@gmail.com>
 Sayam Samal <samal.sayam@gmail.com>
 Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
-Shashank Singh <21bec103@iiitdmj.ac.in>
 Shlok Patel <shlokcpatel2001@gmail.com>
-Shu Chen <shu@zulip.com>
-Shubham Padia <shubham@zulip.com>
-Shubham Padia <shubham@zulip.com> <shubham@glints.com>
 Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
 Steve Howell <showell@zulip.com> <showell30@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@yahoo.com>
@@ -133,19 +106,14 @@ Steve Howell <showell@zulip.com> <showell@zulipchat.com>
 Steve Howell <showell@zulip.com> <steve@humbughq.com>
 Steve Howell <showell@zulip.com> <steve@zulip.com>
 strifel <info@strifel.de>
-Sujal Shah <sujalshah28092004@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com> <133781250+tnmkr@users.noreply.github.com>
 Tim Abbott <tabbott@zulip.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
 Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Tomasz Kolek <tomasz-kolek@o2.pl> <tomasz-kolek@go2.pl>
 Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
 umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
 umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
-Viktor Illmer <1476338+v-ji@users.noreply.github.com>
 Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
 Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
 Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>

.prettierignore

@@ -5,13 +5,7 @@ pnpm-lock.yaml
 /locale
 /templates/**/*.md
 /tools/setup/emoji/emoji_map.json
-/web/third/*
-!/web/third/marked
-/web/third/marked/*
-!/web/third/marked/lib
-/web/third/marked/lib/*
-!/web/third/marked/lib/marked.d.ts
+/web/third
 /zerver/tests/fixtures
 /zerver/webhooks/*/doc.md
-/zerver/webhooks/github/githubsponsors.md
 /zerver/webhooks/*/fixtures

CODE_OF_CONDUCT.md

@@ -66,7 +66,7 @@ organizers may take any action they deem appropriate, up to and including a
 temporary ban or permanent expulsion from the community without warning (and
 without refund in the case of a paid event).
 
-If someone outside the development community (e.g., a user of the Zulip
+If someone outside the development community (e.g. a user of the Zulip
 software) engages in unacceptable behavior that affects someone in the
 community, we still want to know. Even if we don't have direct control over
 the violator, the community organizers can still support the people

CONTRIBUTING.md

@@ -41,9 +41,9 @@ needs doing:
 
 - Bug squashing and feature development on our Python/Django
   [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip),
-  Flutter [mobile app](https://github.com/zulip/zulip-flutter) in beta,
-  or Electron [desktop app](https://github.com/zulip/zulip-desktop).
+  [frontend](https://github.com/zulip/zulip), React Native
+  [mobile app](https://github.com/zulip/zulip-mobile), or Electron
+  [desktop app](https://github.com/zulip/zulip-desktop).
 - Building out our
   [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
 - [Writing an integration](https://zulip.com/api/integrations-overview).
@@ -81,50 +81,28 @@ to help.
   paying special attention to the
   [community norms](https://zulip.com/development-community/#community-norms).
   If you'd like, introduce yourself in
-  [#new members](https://chat.zulip.org/#narrow/channel/95-new-members), using
+  [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
   your name as the topic. Bonus: tell us about your first impressions of
   Zulip, and anything that felt confusing/broken or interesting/helpful as you
   started using the product.
-
 - Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-
-- Set up the development environment for the Zulip codebase you want
-  to work on, and start getting familiar with the code.
-
-  - For the server and web app:
-
-    - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-      getting help in
-      [#provision help](https://chat.zulip.org/#narrow/channel/21-provision-help)
-      if you run into any troubles.
-    - Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
-    - Go through the [new application feature
-      tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
-      how the Zulip codebase is organized and how to find code in it.
-
-  - For the upcoming Flutter-based mobile app:
-    - Set up a development environment following the instructions in
-      [the project README](https://github.com/zulip/zulip-flutter).
-    - Start reading recent commits to see the code we're writing.
-      Use either a [graphical Git viewer][] like `gitk`, or `git log -p`
-      with [the "secret" to reading its output][git-log-secret].
-    - Pick some of the code that appears in those Git commits and
-      that looks interesting. Use your IDE to visit that code
-      and to navigate to related code, reading to see how it works
-      and how the codebase is organized.
-
+- [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
+  getting help in
+  [#provision help](https://chat.zulip.org/#narrow/stream/21-provision-help)
+  if you run into any troubles.
+- Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
+- Go through the [new application feature
+  tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
+  how the Zulip codebase is organized and how to find code in it.
 - Read the [Zulip guide to
   Git](https://zulip.readthedocs.io/en/latest/git/index.html) if you
   are unfamiliar with Git or Zulip's rebase-based Git workflow,
   getting help in [#git
-  help](https://chat.zulip.org/#narrow/channel/44-git-help) if you run
+  help](https://chat.zulip.org/#narrow/stream/44-git-help) if you run
   into any troubles. Even Git experts should read the [Zulip-specific
   Git tools
   page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
 
-[graphical Git viewer]: https://zulip.readthedocs.io/en/latest/git/setup.html#get-a-graphical-client
-[git-log-secret]: https://github.com/zulip/zulip-mobile/blob/main/docs/howto/git.md#git-log-secret
-
 ### Where to look for an issue
 
 Now you're ready to pick your first issue! Zulip has several repositories you
@@ -139,10 +117,7 @@ use the "good first issue" label to tag issues that are especially approachable
 for new contributors.
 
 - [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- Mobile apps: no "help wanted" label, but see the
-  [project board](https://github.com/orgs/zulip/projects/5/views/4)
-  for the upcoming Flutter-based app. Look for issues up through the
-  "Launch" milestone, and that aren't already assigned.
+- [Mobile apps](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 - [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 - [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
 - [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
@@ -217,7 +192,7 @@ issue you're interested in.
 #### In other Zulip repositories
 
 There is no bot for other Zulip repositories
-([`zulip/zulip-flutter`](https://github.com/zulip/zulip-flutter/), etc.). If
+([`zulip/zulip-mobile`](https://github.com/zulip/zulip-mobile/), etc.). If
 you are interested in claiming an issue in one of these repositories, simply
 post a comment on the issue thread saying that you'd like to work on it. There
 is no need to @-mention the issue creator in your comment.
@@ -235,7 +210,7 @@ GitHub issue or pull request.
 
 To get early feedback on any UI changes, we encourage you to post screenshots of
 your work in the [#design
-stream](https://chat.zulip.org/#narrow/channel/101-design) in the [Zulip
+stream](https://chat.zulip.org/#narrow/stream/101-design) in the [Zulip
 development community](https://zulip.com/development-community/)
 
 For more advice, see [What makes a great Zulip
@@ -245,14 +220,10 @@ faster as you build experience.
 
 ### Submitting a pull request
 
-See the [guide on submitting a pull
-request](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html)
-for detailed instructions on how to present your proposed changes to Zulip.
-
-The [pull request review process
-guide](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-explains the stages of review your PR will go through, and offers guidance on
-how to help the review process move forward.
+See the [pull request review
+process](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
+guide for detailed instructions on how to submit a pull request, and information
+on the stages of review your PR will go through.
 
 ### Beyond the first issue
 
@@ -377,7 +348,7 @@ to:
   A link to your organization's website?
 
 You can contact us in the [#feedback stream of the Zulip development
-community](https://chat.zulip.org/#narrow/channel/137-feedback) or
+community](https://chat.zulip.org/#narrow/stream/137-feedback) or
 by emailing [support@zulip.com](mailto:support@zulip.com).
 
 ## Outreach programs
@@ -412,7 +383,7 @@ Here are some ways you can help others find Zulip:
 
 - Star us on GitHub. There are four main repositories:
   [server/web](https://github.com/zulip/zulip),
-  [Flutter mobile](https://github.com/zulip/zulip-flutter),
+  [mobile](https://github.com/zulip/zulip-mobile),
   [desktop](https://github.com/zulip/zulip-desktop), and
   [Python API](https://github.com/zulip/python-zulip-api).
 

README.md

@@ -18,6 +18,7 @@ Come find us on the [development community chat](https://zulip.com/development-c
 [![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
 [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)

Vagrantfile

@@ -15,7 +15,7 @@ Vagrant.configure("2") do |config|
   ubuntu_mirror = ""
   vboxadd_version = nil
 
-  config.vm.box = "bento/ubuntu-22.04"
+  config.vm.box = "bento/ubuntu-20.04"
 
   config.vm.synced_folder ".", "/vagrant", disabled: true
   config.vm.synced_folder ".", "/srv/zulip", docker_consistency: "z"

analytics/lib/counts.py

@@ -1,14 +1,14 @@
 import logging
 import time
 from collections import OrderedDict, defaultdict
-from collections.abc import Callable, Sequence
 from datetime import datetime, timedelta
-from typing import TypeAlias, Union
+from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union
 
 from django.conf import settings
 from django.db import connection, models
+from django.db.models import F
 from psycopg2.sql import SQL, Composable, Identifier, Literal
-from typing_extensions import override
+from typing_extensions import TypeAlias, override
 
 from analytics.models import (
     BaseCount,
@@ -19,20 +19,14 @@ from analytics.models import (
     UserCount,
     installation_epoch,
 )
+from zerver.lib.logging_util import log_to_file
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
 from zerver.models import Message, Realm, RealmAuditLog, Stream, UserActivityInterval, UserProfile
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import (
-        RemoteInstallationCount,
-        RemoteRealm,
-        RemoteRealmCount,
-        RemoteZulipServer,
-    )
-
-
-logger = logging.getLogger("zulip.analytics")
+## Logging setup ##
 
+logger = logging.getLogger("zulip.management")
+log_to_file(logger, settings.ANALYTICS_LOG_PATH)
 
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365 * 1000)
@@ -56,7 +50,7 @@ class CountStat:
         property: str,
         data_collector: "DataCollector",
         frequency: str,
-        interval: timedelta | None = None,
+        interval: Optional[timedelta] = None,
     ) -> None:
         self.property = property
         self.data_collector = data_collector
@@ -73,7 +67,7 @@ class CountStat:
     def __repr__(self) -> str:
         return f"<CountStat: {self.property}>"
 
-    def last_successful_fill(self) -> datetime | None:
+    def last_successful_fill(self) -> Optional[datetime]:
         fillstate = FillState.objects.filter(property=self.property).first()
         if fillstate is None:
             return None
@@ -83,7 +77,7 @@ class CountStat:
 
 
 class LoggingCountStat(CountStat):
-    def __init__(self, property: str, output_table: type[BaseCount], frequency: str) -> None:
+    def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
         CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
 
 
@@ -93,7 +87,7 @@ class DependentCountStat(CountStat):
         property: str,
         data_collector: "DataCollector",
         frequency: str,
-        interval: timedelta | None = None,
+        interval: Optional[timedelta] = None,
         dependencies: Sequence[str] = [],
     ) -> None:
         CountStat.__init__(self, property, data_collector, frequency, interval=interval)
@@ -103,20 +97,19 @@ class DependentCountStat(CountStat):
 class DataCollector:
     def __init__(
         self,
-        output_table: type[BaseCount],
-        pull_function: Callable[[str, datetime, datetime, Realm | None], int] | None,
+        output_table: Type[BaseCount],
+        pull_function: Optional[Callable[[str, datetime, datetime, Optional[Realm]], int]],
     ) -> None:
         self.output_table = output_table
         self.pull_function = pull_function
 
-    def depends_on_realm(self) -> bool:
-        return self.output_table in (UserCount, StreamCount)
-
 
 ## CountStat-level operations ##
 
 
-def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | None = None) -> None:
+def process_count_stat(
+    stat: CountStat, fill_to_time: datetime, realm: Optional[Realm] = None
+) -> None:
     # TODO: The realm argument is not yet supported, in that we don't
     # have a solution for how to update FillState if it is passed.  It
     # exists solely as partial plumbing for when we do fully implement
@@ -158,7 +151,7 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | N
                 return
             fill_to_time = min(fill_to_time, dependency_fill_time)
 
-    currently_filled += stat.time_increment
+    currently_filled = currently_filled + stat.time_increment
     while currently_filled <= fill_to_time:
         logger.info("START %s %s", stat.property, currently_filled)
         start = time.time()
@@ -166,7 +159,7 @@ def process_count_stat(stat: CountStat, fill_to_time: datetime, realm: Realm | N
         do_fill_count_stat_at_hour(stat, currently_filled, realm)
         do_update_fill_state(fill_state, currently_filled, FillState.DONE)
         end = time.time()
-        currently_filled += stat.time_increment
+        currently_filled = currently_filled + stat.time_increment
         logger.info("DONE %s (%dms)", stat.property, (end - start) * 1000)
 
 
@@ -179,7 +172,7 @@ def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int)
 # We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
 # and is time-zone-aware. It is the caller's responsibility to enforce this!
 def do_fill_count_stat_at_hour(
-    stat: CountStat, end_time: datetime, realm: Realm | None = None
+    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
     start_time = end_time - stat.interval
     if not isinstance(stat, LoggingCountStat):
@@ -198,7 +191,7 @@ def do_fill_count_stat_at_hour(
 def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
     if isinstance(stat, LoggingCountStat):
         InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.depends_on_realm():
+        if stat.data_collector.output_table in [UserCount, StreamCount]:
             RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
     else:
         UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
@@ -208,7 +201,7 @@ def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
 
 
 def do_aggregate_to_summary_table(
-    stat: CountStat, end_time: datetime, realm: Realm | None = None
+    stat: CountStat, end_time: datetime, realm: Optional[Realm] = None
 ) -> None:
     cursor = connection.cursor()
 
@@ -219,7 +212,7 @@ def do_aggregate_to_summary_table(
     else:
         realm_clause = SQL("")
 
-    if stat.data_collector.depends_on_realm():
+    if output_table in (UserCount, StreamCount):
         realmcount_query = SQL(
             """
             INSERT INTO analytics_realmcount
@@ -300,9 +293,9 @@ def do_aggregate_to_summary_table(
 
 # called from zerver.actions; should not throw any errors
 def do_increment_logging_stat(
-    model_object_for_bucket: Union[Realm, UserProfile, Stream, "RemoteRealm", "RemoteZulipServer"],
+    zerver_object: Union[Realm, UserProfile, Stream],
     stat: CountStat,
-    subgroup: str | int | bool | None,
+    subgroup: Optional[Union[str, int, bool]],
     event_time: datetime,
     increment: int = 1,
 ) -> None:
@@ -310,100 +303,31 @@ def do_increment_logging_stat(
         return
 
     table = stat.data_collector.output_table
-    id_args: dict[str, int | None] = {}
-    conflict_args: list[str] = []
     if table == RealmCount:
-        assert isinstance(model_object_for_bucket, Realm)
-        id_args = {"realm_id": model_object_for_bucket.id}
-        conflict_args = ["realm_id"]
+        assert isinstance(zerver_object, Realm)
+        id_args: Dict[str, Union[Realm, UserProfile, Stream]] = {"realm": zerver_object}
     elif table == UserCount:
-        assert isinstance(model_object_for_bucket, UserProfile)
-        id_args = {
-            "realm_id": model_object_for_bucket.realm_id,
-            "user_id": model_object_for_bucket.id,
-        }
-        conflict_args = ["user_id"]
-    elif table == StreamCount:
-        assert isinstance(model_object_for_bucket, Stream)
-        id_args = {
-            "realm_id": model_object_for_bucket.realm_id,
-            "stream_id": model_object_for_bucket.id,
-        }
-        conflict_args = ["stream_id"]
-    elif table == RemoteInstallationCount:
-        assert isinstance(model_object_for_bucket, RemoteZulipServer)
-        id_args = {"server_id": model_object_for_bucket.id, "remote_id": None}
-        conflict_args = ["server_id"]
-    elif table == RemoteRealmCount:
-        assert isinstance(model_object_for_bucket, RemoteRealm)
-        # For RemoteRealmCount (e.g. `mobile_pushes_forwarded::day`),
-        # we have no `remote_id` nor `realm_id`, since they are not
-        # imported from the remote server, which is the source of
-        # truth of those two columns.  Their "ON CONFLICT" is thus the
-        # only unique key we have, which is `remote_realm_id`, and not
-        # `server_id` / `realm_id`.
-        id_args = {
-            "server_id": model_object_for_bucket.server_id,
-            "remote_realm_id": model_object_for_bucket.id,
-            "remote_id": None,
-            "realm_id": None,
-        }
-        conflict_args = [
-            "remote_realm_id",
-        ]
-    else:
-        raise AssertionError("Unsupported CountStat output_table")
+        assert isinstance(zerver_object, UserProfile)
+        id_args = {"realm": zerver_object.realm, "user": zerver_object}
+    else:  # StreamCount
+        assert isinstance(zerver_object, Stream)
+        id_args = {"realm": zerver_object.realm, "stream": zerver_object}
 
     if stat.frequency == CountStat.DAY:
         end_time = ceiling_to_day(event_time)
-    elif stat.frequency == CountStat.HOUR:
+    else:  # CountStat.HOUR:
         end_time = ceiling_to_hour(event_time)
-    else:
-        raise AssertionError("Unsupported CountStat frequency")
 
-    is_subgroup: SQL = SQL("NULL")
-    if subgroup is not None:
-        is_subgroup = SQL("NOT NULL")
-        # For backwards consistency, we cast the subgroup to a string
-        # in Python; this emulates the behaviour of `get_or_create`,
-        # which was previously used in this function, and performed
-        # this cast because the `subgroup` column is defined as a
-        # `CharField`.  Omitting this explicit cast causes a subgroup
-        # of the boolean False to be passed as the PostgreSQL false,
-        # which it stringifies as the lower-case `'false'`, not the
-        # initial-case `'False'` if Python stringifies it.
-        #
-        # Other parts of the system (e.g. count_message_by_user_query)
-        # already use PostgreSQL to cast bools to strings, resulting
-        # in `subgroup` values of lower-case `'false'` -- for example
-        # in `messages_sent:is_bot:hour`.  Fixing this inconsistency
-        # via a migration is complicated by these records being
-        # exchanged over the wire from remote servers.
-        subgroup = str(subgroup)
-        conflict_args.append("subgroup")
-
-    id_column_names = SQL(", ").join(map(Identifier, id_args.keys()))
-    id_values = SQL(", ").join(map(Literal, id_args.values()))
-    conflict_columns = SQL(", ").join(map(Identifier, conflict_args))
-
-    sql_query = SQL(
-        """
-        INSERT INTO {table_name}(property, subgroup, end_time, value, {id_column_names})
-        VALUES (%s, %s, %s, %s, {id_values})
-        ON CONFLICT (property, end_time, {conflict_columns})
-        WHERE subgroup IS {is_subgroup}
-        DO UPDATE SET
-            value = {table_name}.value + EXCLUDED.value
-        """
-    ).format(
-        table_name=Identifier(table._meta.db_table),
-        id_column_names=id_column_names,
-        id_values=id_values,
-        conflict_columns=conflict_columns,
-        is_subgroup=is_subgroup,
+    row, created = table._default_manager.get_or_create(
+        property=stat.property,
+        subgroup=subgroup,
+        end_time=end_time,
+        defaults={"value": increment},
+        **id_args,
     )
-    with connection.cursor() as cursor:
-        cursor.execute(sql_query, [stat.property, subgroup, end_time, increment])
+    if not created:
+        row.value = F("value") + increment
+        row.save(update_fields=["value"])
 
 
 def do_drop_all_analytics_tables() -> None:
@@ -424,7 +348,7 @@ def do_drop_single_stat(property: str) -> None:
 
 ## DataCollector-level operations ##
 
-QueryFn: TypeAlias = Callable[[dict[str, Composable]], Composable]
+QueryFn: TypeAlias = Callable[[Dict[str, Composable]], Composable]
 
 
 def do_pull_by_sql_query(
@@ -432,7 +356,7 @@ def do_pull_by_sql_query(
     start_time: datetime,
     end_time: datetime,
     query: QueryFn,
-    group_by: tuple[type[models.Model], str] | None,
+    group_by: Optional[Tuple[Type[models.Model], str]],
 ) -> int:
     if group_by is None:
         subgroup: Composable = SQL("NULL")
@@ -466,12 +390,12 @@ def do_pull_by_sql_query(
 
 
 def sql_data_collector(
-    output_table: type[BaseCount],
+    output_table: Type[BaseCount],
     query: QueryFn,
-    group_by: tuple[type[models.Model], str] | None,
+    group_by: Optional[Tuple[Type[models.Model], str]],
 ) -> DataCollector:
     def pull_function(
-        property: str, start_time: datetime, end_time: datetime, realm: Realm | None = None
+        property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
     ) -> int:
         # The pull function type needs to accept a Realm argument
         # because the 'minutes_active::day' CountStat uses
@@ -484,42 +408,8 @@ def sql_data_collector(
     return DataCollector(output_table, pull_function)
 
 
-def count_upload_space_used_by_realm_query(realm: Realm | None) -> QueryFn:
-    if realm is None:
-        realm_clause: Composable = SQL("")
-    else:
-        realm_clause = SQL("zerver_attachment.realm_id = {} AND").format(Literal(realm.id))
-
-    # Note: This query currently has to go through the entire table,
-    # summing all the sizes of attachments for every realm. This can be improved
-    # by having a query which looks at the latest CountStat for each realm,
-    # and sums it with only the new attachments.
-    # There'd be additional complexity added by the fact that attachments can
-    # also be deleted. Partially this can be accounted for by subtracting
-    # ArchivedAttachment sizes, but there's still the issue of attachments
-    # which can be directly deleted via the API.
-
-    return lambda kwargs: SQL(
-        """
-            INSERT INTO analytics_realmcount (realm_id, property, end_time, value)
-            SELECT
-                zerver_attachment.realm_id,
-                %(property)s,
-                %(time_end)s,
-                COALESCE(SUM(zerver_attachment.size), 0)
-            FROM
-                zerver_attachment
-            WHERE
-                {realm_clause}
-                zerver_attachment.create_time < %(time_end)s
-            GROUP BY
-                zerver_attachment.realm_id
-        """
-    ).format(**kwargs, realm_clause=realm_clause)
-
-
 def do_pull_minutes_active(
-    property: str, start_time: datetime, end_time: datetime, realm: Realm | None = None
+    property: str, start_time: datetime, end_time: datetime, realm: Optional[Realm] = None
 ) -> int:
     user_activity_intervals = (
         UserActivityInterval.objects.filter(
@@ -532,7 +422,7 @@ def do_pull_minutes_active(
         .values_list("user_profile_id", "user_profile__realm_id", "start", "end")
     )
 
-    seconds_active: dict[tuple[int, int], float] = defaultdict(float)
+    seconds_active: Dict[Tuple[int, int], float] = defaultdict(float)
     for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
         if realm is None or realm.id == realm_id:
             start = max(start_time, interval_start)
@@ -554,7 +444,7 @@ def do_pull_minutes_active(
     return len(rows)
 
 
-def count_message_by_user_query(realm: Realm | None) -> QueryFn:
+def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
@@ -587,7 +477,7 @@ def count_message_by_user_query(realm: Realm | None) -> QueryFn:
 
 
 # Note: ignores the group_by / group_by_clause.
-def count_message_type_by_user_query(realm: Realm | None) -> QueryFn:
+def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
@@ -642,7 +532,7 @@ def count_message_type_by_user_query(realm: Realm | None) -> QueryFn:
 # use this also subgroup on UserProfile.is_bot. If in the future there is a
 # stat that counts messages by stream and doesn't need the UserProfile
 # table, consider writing a new query for efficiency.
-def count_message_by_stream_query(realm: Realm | None) -> QueryFn:
+def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
@@ -677,39 +567,67 @@ def count_message_by_stream_query(realm: Realm | None) -> QueryFn:
     ).format(**kwargs, realm_clause=realm_clause)
 
 
-# Hardcodes the query needed for active_users_audit:is_bot:day.
-# Assumes that a user cannot have two RealmAuditLog entries with the
-# same event_time and event_type in [RealmAuditLog.USER_CREATED,
-# USER_DEACTIVATED, etc].  In particular, it's important to ensure
-# that migrations don't cause that to happen.
-def check_realmauditlog_by_user_query(realm: Realm | None) -> QueryFn:
+# Hardcodes the query needed by active_users:is_bot:day, since that is
+# currently the only stat that uses this.
+def count_user_by_realm_query(realm: Optional[Realm]) -> QueryFn:
+    if realm is None:
+        realm_clause: Composable = SQL("")
+    else:
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
+    return lambda kwargs: SQL(
+        """
+    INSERT INTO analytics_realmcount
+        (realm_id, value, property, subgroup, end_time)
+    SELECT
+        zerver_realm.id, count(*), %(property)s, {subgroup}, %(time_end)s
+    FROM zerver_realm
+    JOIN zerver_userprofile
+    ON
+        zerver_realm.id = zerver_userprofile.realm_id
+    WHERE
+        zerver_realm.date_created < %(time_end)s AND
+        zerver_userprofile.date_joined >= %(time_start)s AND
+        zerver_userprofile.date_joined < %(time_end)s AND
+        {realm_clause}
+        zerver_userprofile.is_active = TRUE
+    GROUP BY zerver_realm.id {group_by_clause}
+"""
+    ).format(**kwargs, realm_clause=realm_clause)
+
+
+# Currently hardcodes the query needed for active_users_audit:is_bot:day.
+# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
+# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
+# In particular, it's important to ensure that migrations don't cause that to happen.
+def check_realmauditlog_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
         realm_clause = SQL("realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
-    INSERT INTO analytics_realmcount
-        (realm_id, value, property, subgroup, end_time)
+    INSERT INTO analytics_usercount
+        (user_id, realm_id, value, property, subgroup, end_time)
     SELECT
-        zerver_userprofile.realm_id, count(*), %(property)s, {subgroup}, %(time_end)s
-    FROM zerver_userprofile
+        ral1.modified_user_id, ral1.realm_id, 1, %(property)s, {subgroup}, %(time_end)s
+    FROM zerver_realmauditlog ral1
     JOIN (
-            SELECT DISTINCT ON (modified_user_id)
-                    modified_user_id, event_type
-            FROM
-                    zerver_realmauditlog
-            WHERE
-                    event_type IN ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
-                    {realm_clause}
-                    event_time < %(time_end)s
-            ORDER BY
-                    modified_user_id,
-                    event_time DESC
-    ) last_user_event ON last_user_event.modified_user_id = zerver_userprofile.id
+        SELECT modified_user_id, max(event_time) AS max_event_time
+        FROM zerver_realmauditlog
+        WHERE
+            event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
+            {realm_clause}
+            event_time < %(time_end)s
+        GROUP BY modified_user_id
+    ) ral2
+    ON
+        ral1.event_time = max_event_time AND
+        ral1.modified_user_id = ral2.modified_user_id
+    JOIN zerver_userprofile
+    ON
+        ral1.modified_user_id = zerver_userprofile.id
     WHERE
-        last_user_event.event_type in ({user_created}, {user_activated}, {user_reactivated})
-    GROUP BY zerver_userprofile.realm_id {group_by_clause}
+        ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
     """
     ).format(
         **kwargs,
@@ -721,7 +639,7 @@ def check_realmauditlog_by_user_query(realm: Realm | None) -> QueryFn:
     )
 
 
-def check_useractivityinterval_by_user_query(realm: Realm | None) -> QueryFn:
+def check_useractivityinterval_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
@@ -745,7 +663,7 @@ def check_useractivityinterval_by_user_query(realm: Realm | None) -> QueryFn:
     ).format(**kwargs, realm_clause=realm_clause)
 
 
-def count_realm_active_humans_query(realm: Realm | None) -> QueryFn:
+def count_realm_active_humans_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
@@ -755,45 +673,29 @@ def count_realm_active_humans_query(realm: Realm | None) -> QueryFn:
     INSERT INTO analytics_realmcount
         (realm_id, value, property, subgroup, end_time)
     SELECT
-            active_usercount.realm_id, count(*), %(property)s, NULL, %(time_end)s
+        usercount1.realm_id, count(*), %(property)s, NULL, %(time_end)s
     FROM (
-            SELECT
-                    realm_id,
-                    user_id
-            FROM
-                    analytics_usercount
-            WHERE
-                    property = '15day_actives::day'
-                    {realm_clause}
-                    AND end_time = %(time_end)s
-    ) active_usercount
-    JOIN zerver_userprofile ON active_usercount.user_id = zerver_userprofile.id
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = 'active_users_audit:is_bot:day' AND
+            subgroup = 'false' AND
+            {realm_clause}
+            end_time = %(time_end)s
+    ) usercount1
     JOIN (
-            SELECT DISTINCT ON (modified_user_id)
-                    modified_user_id, event_type
-            FROM
-                    zerver_realmauditlog
-            WHERE
-                    event_type IN ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated})
-                    AND event_time < %(time_end)s
-            ORDER BY
-                    modified_user_id,
-                    event_time DESC
-    ) last_user_event ON last_user_event.modified_user_id = active_usercount.user_id
-    WHERE
-            NOT zerver_userprofile.is_bot
-            AND event_type IN ({user_created}, {user_activated}, {user_reactivated})
-    GROUP BY
-            active_usercount.realm_id
+        SELECT realm_id, user_id
+        FROM analytics_usercount
+        WHERE
+            property = '15day_actives::day' AND
+            {realm_clause}
+            end_time = %(time_end)s
+    ) usercount2
+    ON
+        usercount1.user_id = usercount2.user_id
+    GROUP BY usercount1.realm_id
 """
-    ).format(
-        **kwargs,
-        user_created=Literal(RealmAuditLog.USER_CREATED),
-        user_activated=Literal(RealmAuditLog.USER_ACTIVATED),
-        user_deactivated=Literal(RealmAuditLog.USER_DEACTIVATED),
-        user_reactivated=Literal(RealmAuditLog.USER_REACTIVATED),
-        realm_clause=realm_clause,
-    )
+    ).format(**kwargs, realm_clause=realm_clause)
 
 
 # Currently unused and untested
@@ -816,7 +718,7 @@ count_stream_by_realm_query = lambda kwargs: SQL(
 ).format(**kwargs)
 
 
-def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
+def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
     ## CountStat declarations ##
 
     count_stats_ = [
@@ -849,19 +751,39 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
             ),
             CountStat.DAY,
         ),
-        # Counts the number of active users in the UserProfile.is_active sense.
+        # Number of users stats
+        # Stats that count the number of active users in the UserProfile.is_active sense.
+        # 'active_users_audit:is_bot:day' is the canonical record of which users were
+        # active on which days (in the UserProfile.is_active sense).
         # Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
         CountStat(
             "active_users_audit:is_bot:day",
             sql_data_collector(
-                RealmCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
+                UserCount, check_realmauditlog_by_user_query(realm), (UserProfile, "is_bot")
             ),
             CountStat.DAY,
         ),
+        # Important note: LoggingCountStat objects aren't passed the
+        # Realm argument, because by nature they have a logging
+        # structure, not a pull-from-database structure, so there's no
+        # way to compute them for a single realm after the fact (the
+        # use case for passing a Realm argument).
+        # Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
+        # In RealmCount, 'active_users_audit:is_bot:day' should be the partial
+        # sum sequence of 'active_users_log:is_bot:day', for any realm that
+        # started after the latter stat was introduced.
+        LoggingCountStat("active_users_log:is_bot:day", RealmCount, CountStat.DAY),
+        # Another sanity check on 'active_users_audit:is_bot:day'. Is only an
+        # approximation, e.g. if a user is deactivated between the end of the
+        # day and when this stat is run, they won't be counted. However, is the
+        # simplest of the three to inspect by hand.
         CountStat(
-            "upload_quota_used_bytes::day",
-            sql_data_collector(RealmCount, count_upload_space_used_by_realm_query(realm), None),
+            "active_users:is_bot:day",
+            sql_data_collector(
+                RealmCount, count_user_by_realm_query(realm), (UserProfile, "is_bot")
+            ),
             CountStat.DAY,
+            interval=TIMEDELTA_MAX,
         ),
         # Messages read stats.  messages_read::hour is the total
         # number of messages read, whereas
@@ -895,16 +817,8 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
         CountStat(
             "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
         ),
-        # Tracks the number of push notifications requested by the server.
-        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
-        LoggingCountStat(
-            "mobile_pushes_sent::day",
-            RealmCount,
-            CountStat.DAY,
-        ),
         # Rate limiting stats
-        # Used to limit the number of invitation emails sent by a realm.
-        # Included in LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER.
+        # Used to limit the number of invitation emails sent by a realm
         LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
         # Dependent stats
         # Must come after their dependencies.
@@ -913,83 +827,12 @@ def get_count_stats(realm: Realm | None = None) -> dict[str, CountStat]:
             "realm_active_humans::day",
             sql_data_collector(RealmCount, count_realm_active_humans_query(realm), None),
             CountStat.DAY,
-            dependencies=["15day_actives::day"],
+            dependencies=["active_users_audit:is_bot:day", "15day_actives::day"],
         ),
     ]
 
-    if settings.ZILENCER_ENABLED:
-        # See also the remote_installation versions of these in REMOTE_INSTALLATION_COUNT_STATS.
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_received::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_forwarded::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-
     return OrderedDict((stat.property, stat) for stat in count_stats_)
 
 
-# These properties are tracked by the bouncer itself and therefore syncing them
-# from a remote server should not be allowed - or the server would be able to interfere
-# with our data.
-BOUNCER_ONLY_REMOTE_COUNT_STAT_PROPERTIES = [
-    "mobile_pushes_received::day",
-    "mobile_pushes_forwarded::day",
-]
-
-# LoggingCountStats with a daily duration and that are directly stored on
-# the RealmCount table (instead of via aggregation in process_count_stat),
-# can be in a state, after the hourly cron job to update analytics counts,
-# where the logged value will be live-updated later (as the end time for
-# the stat is still in the future). As these logging counts are designed
-# to be used on the self-hosted installation for either debugging or rate
-# limiting, sending these incomplete counts to the bouncer has low value.
-LOGGING_COUNT_STAT_PROPERTIES_NOT_SENT_TO_BOUNCER = {
-    "invites_sent::day",
-    "mobile_pushes_sent::day",
-    "active_users_log:is_bot:day",
-    "active_users:is_bot:day",
-}
-
 # To avoid refactoring for now COUNT_STATS can be used as before
 COUNT_STATS = get_count_stats()
-
-REMOTE_INSTALLATION_COUNT_STATS = OrderedDict()
-
-if settings.ZILENCER_ENABLED:
-    # REMOTE_INSTALLATION_COUNT_STATS contains duplicates of the
-    # RemoteRealmCount stats declared above; it is necessary because
-    # pre-8.0 servers do not send the fields required to identify a
-    # RemoteRealm.
-
-    # Tracks the number of push notifications requested to be sent
-    # by a remote server.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_received::day"] = LoggingCountStat(
-        "mobile_pushes_received::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-    # Tracks the number of push notifications successfully sent to
-    # mobile devices, as requested by the remote server. Therefore
-    # this should be less than or equal to mobile_pushes_received -
-    # with potential tiny offsets resulting from a request being
-    # *received* by the bouncer right before midnight, but *sent* to
-    # the mobile device right after midnight. This would cause the
-    # increments to happen to CountStat records for different days.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_forwarded::day"] = LoggingCountStat(
-        "mobile_pushes_forwarded::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-
-ALL_COUNT_STATS = OrderedDict(
-    list(COUNT_STATS.items()) + list(REMOTE_INSTALLATION_COUNT_STATS.items())
-)

analytics/lib/fixtures.py

@@ -1,5 +1,6 @@
 from math import sqrt
 from random import Random
+from typing import List
 
 from analytics.lib.counts import CountStat
 
@@ -15,7 +16,7 @@ def generate_time_series_data(
     frequency: str = CountStat.DAY,
     partial_sum: bool = False,
     random_seed: int = 26,
-) -> list[int]:
+) -> List[int]:
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
 
@@ -59,7 +60,9 @@ def generate_time_series_data(
             f"Must be generating at least 2 data points. Currently generating {length}"
         )
     growth_base = growth ** (1.0 / (length - 1))
-    values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
+    values_no_noise = [
+        seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)
+    ]
 
     noise_scalars = [rng.gauss(0, 1)]
     for i in range(1, length):
@@ -69,7 +72,7 @@ def generate_time_series_data(
 
     values = [
         0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
-        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays, strict=False)
+        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)
     ]
     if partial_sum:
         for i in range(1, length):

analytics/lib/time_utils.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+from typing import List, Optional
 
 from analytics.lib.counts import CountStat
 from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
@@ -9,8 +10,8 @@ from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
 def time_range(
-    start: datetime, end: datetime, frequency: str, min_length: int | None
-) -> list[datetime]:
+    start: datetime, end: datetime, frequency: str, min_length: Optional[int]
+) -> List[datetime]:
     verify_UTC(start)
     verify_UTC(end)
     if frequency == CountStat.HOUR:

analytics/management/commands/check_analytics_state.py

@@ -1,14 +1,14 @@
-from dataclasses import dataclass
+import os
+import time
 from datetime import timedelta
-from typing import Any, Literal
+from typing import Any, Dict
 
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, CountStat
+from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.models import installation_epoch
-from scripts.lib.zulip_tools import atomic_nagios_write
-from zerver.lib.management import ZulipBaseCommand
 from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
 from zerver.models import Realm
 
@@ -20,13 +20,7 @@ states = {
 }
 
 
-@dataclass
-class NagiosResult:
-    status: Literal["ok", "warning", "critical", "unknown"]
-    message: str
-
-
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Checks FillState table.
 
     Run as a cron job that runs every hour."""
@@ -34,24 +28,30 @@ class Command(ZulipBaseCommand):
     @override
     def handle(self, *args: Any, **options: Any) -> None:
         fill_state = self.get_fill_state()
-        atomic_nagios_write("check-analytics-state", fill_state.status, fill_state.message)
+        status = fill_state["status"]
+        message = fill_state["message"]
 
-    def get_fill_state(self) -> NagiosResult:
+        state_file_path = "/var/lib/nagios_state/check-analytics-state"
+        state_file_tmp = state_file_path + "-tmp"
+
+        with open(state_file_tmp, "w") as f:
+            f.write(f"{int(time.time())}|{status}|{states[status]}|{message}\n")
+        os.rename(state_file_tmp, state_file_path)
+
+    def get_fill_state(self) -> Dict[str, Any]:
         if not Realm.objects.exists():
-            return NagiosResult(status="ok", message="No realms exist, so not checking FillState.")
+            return {"status": 0, "message": "No realms exist, so not checking FillState."}
 
         warning_unfilled_properties = []
         critical_unfilled_properties = []
-        for property, stat in ALL_COUNT_STATS.items():
+        for property, stat in COUNT_STATS.items():
             last_fill = stat.last_successful_fill()
             if last_fill is None:
                 last_fill = installation_epoch()
             try:
                 verify_UTC(last_fill)
             except TimeZoneNotUTCError:
-                return NagiosResult(
-                    status="critical", message=f"FillState not in UTC for {property}"
-                )
+                return {"status": 2, "message": f"FillState not in UTC for {property}"}
 
             if stat.frequency == CountStat.DAY:
                 floor_function = floor_to_day
@@ -63,10 +63,10 @@ class Command(ZulipBaseCommand):
                 critical_threshold = timedelta(minutes=150)
 
             if floor_function(last_fill) != last_fill:
-                return NagiosResult(
-                    status="critical",
-                    message=f"FillState not on {stat.frequency} boundary for {property}",
-                )
+                return {
+                    "status": 2,
+                    "message": f"FillState not on {stat.frequency} boundary for {property}",
+                }
 
             time_to_last_fill = timezone_now() - last_fill
             if time_to_last_fill > critical_threshold:
@@ -75,18 +75,18 @@ class Command(ZulipBaseCommand):
                 warning_unfilled_properties.append(property)
 
         if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return NagiosResult(status="ok", message="FillState looks fine.")
+            return {"status": 0, "message": "FillState looks fine."}
         if len(critical_unfilled_properties) == 0:
-            return NagiosResult(
-                status="warning",
-                message="Missed filling {} once.".format(
+            return {
+                "status": 1,
+                "message": "Missed filling {} once.".format(
                     ", ".join(warning_unfilled_properties),
                 ),
-            )
-        return NagiosResult(
-            status="critical",
-            message="Missed filling {} once. Missed filling {} at least twice.".format(
+            }
+        return {
+            "status": 2,
+            "message": "Missed filling {} once. Missed filling {} at least twice.".format(
                 ", ".join(warning_unfilled_properties),
                 ", ".join(critical_unfilled_properties),
             ),
-        )
+        }

analytics/management/commands/clear_analytics_tables.py

@@ -1,14 +1,13 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from django.core.management.base import CommandError
+from django.core.management.base import BaseCommand, CommandError
 from typing_extensions import override
 
 from analytics.lib.counts import do_drop_all_analytics_tables
-from zerver.lib.management import ZulipBaseCommand
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Clear analytics tables."""
 
     @override

analytics/management/commands/clear_single_stat.py

@@ -1,14 +1,13 @@
 from argparse import ArgumentParser
 from typing import Any
 
-from django.core.management.base import CommandError
+from django.core.management.base import BaseCommand, CommandError
 from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
-from zerver.lib.management import ZulipBaseCommand
+from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Clear analytics tables."""
 
     @override
@@ -19,7 +18,7 @@ class Command(ZulipBaseCommand):
     @override
     def handle(self, *args: Any, **options: Any) -> None:
         property = options["property"]
-        if property not in ALL_COUNT_STATS:
+        if property not in COUNT_STATS:
             raise CommandError(f"Invalid property: {property}")
         if not options["force"]:
             raise CommandError("No action taken. Use --force.")

analytics/management/commands/populate_analytics_db.py

@@ -1,10 +1,11 @@
-from collections.abc import Mapping
+import os
 from datetime import timedelta
-from typing import Any, TypeAlias
+from typing import Any, Dict, List, Mapping, Type, Union
 
 from django.core.files.uploadedfile import UploadedFile
+from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
+from typing_extensions import TypeAlias, override
 
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
@@ -20,25 +21,23 @@ from analytics.models import (
 from zerver.actions.create_realm import do_create_realm
 from zerver.actions.users import do_change_user_role
 from zerver.lib.create_user import create_user
-from zerver.lib.management import ZulipBaseCommand
 from zerver.lib.storage import static_path
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
 from zerver.lib.upload import upload_message_attachment_from_request
 from zerver.models import (
     Client,
-    NamedUserGroup,
     Realm,
     RealmAuditLog,
     Recipient,
     Stream,
     Subscription,
+    UserGroup,
     UserProfile,
 )
-from zerver.models.groups import SystemGroups
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
 
     DAYS_OF_DATA = 100
@@ -54,7 +53,7 @@ class Command(ZulipBaseCommand):
         spikiness: float,
         holiday_rate: float = 0,
         partial_sum: bool = False,
-    ) -> list[int]:
+    ) -> List[int]:
         self.random_seed += 1
         return generate_time_series_data(
             days=self.DAYS_OF_DATA,
@@ -114,8 +113,8 @@ class Command(ZulipBaseCommand):
             force_date_joined=installation_time,
         )
 
-        administrators_user_group = NamedUserGroup.objects.get(
-            name=SystemGroups.ADMINISTRATORS, realm=realm, is_system_group=True
+        administrators_user_group = UserGroup.objects.get(
+            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=realm, is_system_group=True
         )
         stream = Stream.objects.create(
             name="all",
@@ -145,21 +144,23 @@ class Command(ZulipBaseCommand):
 
         # Create an attachment in the database for set_storage_space_used_statistic.
         IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
+        file_info = os.stat(IMAGE_FILE_PATH)
+        file_size = file_info.st_size
         with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock)
+            upload_message_attachment_from_request(UploadedFile(fp), shylock, file_size)
 
-        FixtureData: TypeAlias = Mapping[str | int | None, list[int]]
+        FixtureData: TypeAlias = Mapping[Union[str, int, None], List[int]]
 
         def insert_fixture_data(
             stat: CountStat,
             fixture_data: FixtureData,
-            table: type[BaseCount],
+            table: Type[BaseCount],
         ) -> None:
             end_times = time_range(
                 last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
             )
             if table == InstallationCount:
-                id_args: dict[str, Any] = {}
+                id_args: Dict[str, Any] = {}
             if table == RealmCount:
                 id_args = {"realm": realm}
             if table == UserCount:
@@ -176,7 +177,7 @@ class Command(ZulipBaseCommand):
                         value=value,
                         **id_args,
                     )
-                    for end_time, value in zip(end_times, values, strict=False)
+                    for end_time, value in zip(end_times, values)
                     if value != 0
                 )
 
@@ -283,7 +284,6 @@ class Command(ZulipBaseCommand):
         android, created = Client.objects.get_or_create(name="ZulipAndroid")
         iOS, created = Client.objects.get_or_create(name="ZulipiOS")
         react_native, created = Client.objects.get_or_create(name="ZulipMobile")
-        flutter, created = Client.objects.get_or_create(name="ZulipFlutter")
         API, created = Client.objects.get_or_create(name="API: Python")
         zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
         unused, created = Client.objects.get_or_create(name="unused")
@@ -301,7 +301,6 @@ class Command(ZulipBaseCommand):
             android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
             iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
             react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
             zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
             unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
@@ -313,7 +312,6 @@ class Command(ZulipBaseCommand):
             old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
             android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
             iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
             API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
             zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
@@ -331,7 +329,7 @@ class Command(ZulipBaseCommand):
             "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
         }
         insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data: Mapping[int | str | None, list[int]] = {
+        stream_data: Mapping[Union[int, str, None], List[int]] = {
             "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
             "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
         }

analytics/management/commands/update_analytics_counts.py

@@ -1,22 +1,23 @@
-import hashlib
+import os
 import time
 from argparse import ArgumentParser
 from datetime import timezone
-from typing import Any
+from typing import Any, Dict
 
 from django.conf import settings
+from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
 from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
-from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from scripts.lib.zulip_tools import ENDC, WARNING
+from zerver.lib.remote_server import send_analytics_to_push_bouncer
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Fills Analytics tables.
 
     Run as a cron job that runs every hour."""
@@ -39,11 +40,22 @@ class Command(ZulipBaseCommand):
         )
 
     @override
-    @abort_unless_locked
     def handle(self, *args: Any, **options: Any) -> None:
-        self.run_update_analytics_counts(options)
+        try:
+            os.mkdir(settings.ANALYTICS_LOCK_DIR)
+        except OSError:
+            print(
+                f"{WARNING}Analytics lock {settings.ANALYTICS_LOCK_DIR} is unavailable;"
+                f" exiting.{ENDC}"
+            )
+            return
 
-    def run_update_analytics_counts(self, options: dict[str, Any]) -> None:
+        try:
+            self.run_update_analytics_counts(options)
+        finally:
+            os.rmdir(settings.ANALYTICS_LOCK_DIR)
+
+    def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
         # installation_epoch relies on there being at least one realm; we
         # shouldn't run the analytics code if that condition isn't satisfied
         if not Realm.objects.exists():
@@ -62,9 +74,9 @@ class Command(ZulipBaseCommand):
         fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
 
         if options["stat"] is not None:
-            stats = [ALL_COUNT_STATS[options["stat"]]]
+            stats = [COUNT_STATS[options["stat"]]]
         else:
-            stats = list(ALL_COUNT_STATS.values())
+            stats = list(COUNT_STATS.values())
 
         logger.info("Starting updating analytics counts through %s", fill_to_time)
         if options["verbose"]:
@@ -83,17 +95,5 @@ class Command(ZulipBaseCommand):
             )
         logger.info("Finished updating analytics counts through %s", fill_to_time)
 
-        if should_send_analytics_data():
-            # Based on the specific value of the setting, the exact details to send
-            # will be decided. However, we proceed just based on this not being falsey.
-
-            # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
-            # that each server will report in at a somewhat consistent time.
-            assert settings.ZULIP_ORG_ID
-            delay = int.from_bytes(
-                hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
-            ) % (60 * 10)
-            logger.info("Sleeping %d seconds before reporting...", delay)
-            time.sleep(delay)
-
-            send_server_data_to_push_bouncer(consider_usage_statistics=True)
+        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
+            send_analytics_to_push_bouncer()

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -1,114 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0016_unique_constraint_when_subgroup_null"),
-    ]
-
-    # If the server was installed between 7.0 and 7.4 (or main between
-    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
-    # running 7.5 or 7807bff52635 or higher) are never used, because
-    # they contain an improper cast
-    # (https://code.djangoproject.com/ticket/34840).
-    #
-    # We regenerate the indexes here, by dropping and re-creating
-    # them, so that we know that they are properly formed.
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/migrations/0018_remove_usercount_active_users_audit.py

@@ -1,15 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0017_regenerate_partial_indexes"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            "DELETE FROM analytics_usercount WHERE property = 'active_users_audit:is_bot:day'"
-        )
-    ]

analytics/migrations/0019_remove_unused_counts.py

@@ -1,26 +0,0 @@
-from django.db import migrations
-
-REMOVED_COUNTS = (
-    "active_users_log:is_bot:day",
-    "active_users:is_bot:day",
-)
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0018_remove_usercount_active_users_audit"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            [
-                ("DELETE FROM analytics_realmcount WHERE property IN %s", (REMOVED_COUNTS,)),
-                (
-                    "DELETE FROM analytics_installationcount WHERE property IN %s",
-                    (REMOVED_COUNTS,),
-                ),
-            ]
-        )
-    ]

analytics/migrations/0020_alter_installationcount_id_alter_realmcount_id_and_more.py

@@ -1,40 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    atomic = False
-
-    dependencies = [
-        ("analytics", "0019_remove_unused_counts"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/migrations/0021_alter_fillstate_id.py

@@ -1,17 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="fillstate",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/models.py

@@ -1,7 +1,7 @@
 # https://github.com/typeddjango/django-stubs/issues/1698
 # mypy: disable-error-code="explicit-override"
 
-from datetime import datetime
+import datetime
 
 from django.db import models
 from django.db.models import Q, UniqueConstraint
@@ -27,7 +27,7 @@ class FillState(models.Model):
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime:
+def installation_epoch() -> datetime.datetime:
     earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
         "date_created__min"
     ]

analytics/tests/test_activity_views.py

@@ -0,0 +1,53 @@
+from unittest import mock
+
+from django.utils.timezone import now as timezone_now
+
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.models import Client, UserActivity, UserProfile
+
+
+class ActivityTest(ZulipTestCase):
+    @mock.patch("stripe.Customer.list", return_value=[])
+    def test_activity(self, unused_mock: mock.Mock) -> None:
+        self.login("hamlet")
+        client, _ = Client.objects.get_or_create(name="website")
+        query = "/json/messages/flags"
+        last_visit = timezone_now()
+        count = 150
+        for activity_user_profile in UserProfile.objects.all():
+            UserActivity.objects.get_or_create(
+                user_profile=activity_user_profile,
+                client=client,
+                query=query,
+                count=count,
+                last_visit=last_visit,
+            )
+
+        # Fails when not staff
+        result = self.client_get("/activity")
+        self.assertEqual(result.status_code, 302)
+
+        user_profile = self.example_user("hamlet")
+        user_profile.is_staff = True
+        user_profile.save(update_fields=["is_staff"])
+
+        with self.assert_database_query_count(12):
+            result = self.client_get("/activity")
+            self.assertEqual(result.status_code, 200)
+
+        with self.assert_database_query_count(4):
+            result = self.client_get("/activity/remote")
+            self.assertEqual(result.status_code, 200)
+
+        with self.assert_database_query_count(4):
+            result = self.client_get("/activity/integrations")
+            self.assertEqual(result.status_code, 200)
+
+        with self.assert_database_query_count(8):
+            result = self.client_get("/realm_activity/zulip/")
+            self.assertEqual(result.status_code, 200)
+
+        iago = self.example_user("iago")
+        with self.assert_database_query_count(5):
+            result = self.client_get(f"/user_activity/{iago.id}/")
+            self.assertEqual(result.status_code, 200)

analytics/tests/test_counts.py

@@ -1,10 +1,8 @@
-from collections.abc import Iterator
-from contextlib import AbstractContextManager, ExitStack, contextmanager
 from datetime import datetime, timedelta, timezone
-from typing import Any
+from typing import Any, Dict, List, Optional, Tuple, Type
 from unittest import mock
 
-import time_machine
+import orjson
 from django.apps import apps
 from django.db import models
 from django.db.models import Sum
@@ -41,7 +39,11 @@ from zerver.actions.create_user import (
     do_create_user,
     do_reactivate_user,
 )
-from zerver.actions.invites import do_invite_users, do_revoke_user_invite, do_send_user_invite_email
+from zerver.actions.invites import (
+    do_invite_users,
+    do_resend_user_invite_email,
+    do_revoke_user_invite,
+)
 from zerver.actions.message_flags import (
     do_mark_all_as_read,
     do_mark_stream_messages_as_read,
@@ -51,43 +53,26 @@ from zerver.actions.user_activity import update_user_activity_interval
 from zerver.actions.users import do_deactivate_user
 from zerver.lib.create_user import create_user
 from zerver.lib.exceptions import InvitationError
-from zerver.lib.push_notifications import (
-    get_message_payload_apns,
-    get_message_payload_gcm,
-    hex_to_b64,
-)
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import activate_push_notification_service
-from zerver.lib.timestamp import TimeZoneNotUTCError, ceiling_to_day, floor_to_day
+from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day
 from zerver.lib.topic import DB_TOPIC_NAME
-from zerver.lib.user_counts import realm_user_count_by_role
 from zerver.lib.utils import assert_is_not_none
 from zerver.models import (
     Client,
-    DirectMessageGroup,
+    Huddle,
     Message,
-    NamedUserGroup,
     PreregistrationUser,
     Realm,
     RealmAuditLog,
     Recipient,
     Stream,
     UserActivityInterval,
+    UserGroup,
     UserProfile,
+    get_client,
+    get_user,
+    is_cross_realm_bot_email,
 )
-from zerver.models.clients import get_client
-from zerver.models.groups import SystemGroups
-from zerver.models.messages import Attachment
-from zerver.models.scheduled_jobs import NotificationTriggers
-from zerver.models.users import get_user, is_cross_realm_bot_email
-from zilencer.models import (
-    RemoteInstallationCount,
-    RemotePushDeviceToken,
-    RemoteRealm,
-    RemoteRealmCount,
-    RemoteZulipServer,
-)
-from zilencer.views import get_last_id_from_server
 
 
 class AnalyticsTestCase(ZulipTestCase):
@@ -103,23 +88,17 @@ class AnalyticsTestCase(ZulipTestCase):
         self.default_realm = do_create_realm(
             string_id="realmtest", name="Realm Test", date_created=self.TIME_ZERO - 2 * self.DAY
         )
-        self.administrators_user_group = NamedUserGroup.objects.get(
-            name=SystemGroups.ADMINISTRATORS,
-            realm=self.default_realm,
-            is_system_group=True,
+        self.administrators_user_group = UserGroup.objects.get(
+            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=self.default_realm, is_system_group=True
         )
 
         # used to generate unique names in self.create_*
         self.name_counter = 100
         # used as defaults in self.assert_table_count
-        self.current_property: str | None = None
-
-        # Delete RemoteRealm registrations to have a clean slate - the relevant
-        # tests want to construct this from scratch.
-        RemoteRealm.objects.all().delete()
+        self.current_property: Optional[str] = None
 
     # Lightweight creation of users, streams, and messages
-    def create_user(self, skip_auditlog: bool = False, **kwargs: Any) -> UserProfile:
+    def create_user(self, **kwargs: Any) -> UserProfile:
         self.name_counter += 1
         defaults = {
             "email": f"user{self.name_counter}@domain.tld",
@@ -132,12 +111,12 @@ class AnalyticsTestCase(ZulipTestCase):
         for key, value in defaults.items():
             kwargs[key] = kwargs.get(key, value)
         kwargs["delivery_email"] = kwargs["email"]
-        with time_machine.travel(kwargs["date_joined"], tick=False):
-            pass_kwargs: dict[str, Any] = {}
+        with mock.patch("zerver.lib.create_user.timezone_now", return_value=kwargs["date_joined"]):
+            pass_kwargs: Dict[str, Any] = {}
             if kwargs["is_bot"]:
                 pass_kwargs["bot_type"] = UserProfile.DEFAULT_BOT
                 pass_kwargs["bot_owner"] = None
-            user = create_user(
+            return create_user(
                 kwargs["email"],
                 "password",
                 kwargs["realm"],
@@ -146,20 +125,8 @@ class AnalyticsTestCase(ZulipTestCase):
                 role=UserProfile.ROLE_REALM_ADMINISTRATOR,
                 **pass_kwargs,
             )
-            if not skip_auditlog:
-                RealmAuditLog.objects.create(
-                    realm=kwargs["realm"],
-                    acting_user=None,
-                    modified_user=user,
-                    event_type=RealmAuditLog.USER_CREATED,
-                    event_time=kwargs["date_joined"],
-                    extra_data={
-                        RealmAuditLog.ROLE_COUNT: realm_user_count_by_role(kwargs["realm"])
-                    },
-                )
-            return user
 
-    def create_stream_with_recipient(self, **kwargs: Any) -> tuple[Stream, Recipient]:
+    def create_stream_with_recipient(self, **kwargs: Any) -> Tuple[Stream, Recipient]:
         self.name_counter += 1
         defaults = {
             "name": f"stream name {self.name_counter}",
@@ -175,13 +142,13 @@ class AnalyticsTestCase(ZulipTestCase):
         stream.save(update_fields=["recipient"])
         return stream, recipient
 
-    def create_huddle_with_recipient(self, **kwargs: Any) -> tuple[DirectMessageGroup, Recipient]:
+    def create_huddle_with_recipient(self, **kwargs: Any) -> Tuple[Huddle, Recipient]:
         self.name_counter += 1
         defaults = {"huddle_hash": f"hash{self.name_counter}"}
         for key, value in defaults.items():
             kwargs[key] = kwargs.get(key, value)
-        huddle = DirectMessageGroup.objects.create(**kwargs)
-        recipient = Recipient.objects.create(type_id=huddle.id, type=Recipient.DIRECT_MESSAGE_GROUP)
+        huddle = Huddle.objects.create(**kwargs)
+        recipient = Recipient.objects.create(type_id=huddle.id, type=Recipient.HUDDLE)
         huddle.recipient = recipient
         huddle.save(update_fields=["recipient"])
         return huddle, recipient
@@ -204,33 +171,15 @@ class AnalyticsTestCase(ZulipTestCase):
             kwargs[key] = kwargs.get(key, value)
         return Message.objects.create(**kwargs)
 
-    def create_attachment(
-        self,
-        user_profile: UserProfile,
-        filename: str,
-        size: int,
-        create_time: datetime,
-        content_type: str,
-    ) -> Attachment:
-        return Attachment.objects.create(
-            file_name=filename,
-            path_id=f"foo/bar/{filename}",
-            owner=user_profile,
-            realm=user_profile.realm,
-            size=size,
-            create_time=create_time,
-            content_type=content_type,
-        )
-
     # kwargs should only ever be a UserProfile or Stream.
     def assert_table_count(
         self,
-        table: type[BaseCount],
+        table: Type[BaseCount],
         value: int,
-        property: str | None = None,
-        subgroup: str | None = None,
+        property: Optional[str] = None,
+        subgroup: Optional[str] = None,
         end_time: datetime = TIME_ZERO,
-        realm: Realm | None = None,
+        realm: Optional[Realm] = None,
         **kwargs: models.Model,
     ) -> None:
         if property is None:
@@ -247,7 +196,7 @@ class AnalyticsTestCase(ZulipTestCase):
         self.assertEqual(queryset.values_list("value", flat=True)[0], value)
 
     def assertTableState(
-        self, table: type[BaseCount], arg_keys: list[str], arg_values: list[list[object]]
+        self, table: Type[BaseCount], arg_keys: List[str], arg_values: List[List[object]]
     ) -> None:
         """Assert that the state of a *Count table is what it should be.
 
@@ -277,15 +226,12 @@ class AnalyticsTestCase(ZulipTestCase):
             "value": 1,
         }
         for values in arg_values:
-            kwargs: dict[str, Any] = {}
+            kwargs: Dict[str, Any] = {}
             for i in range(len(values)):
                 kwargs[arg_keys[i]] = values[i]
             for key, value in defaults.items():
                 kwargs[key] = kwargs.get(key, value)
-            if (
-                table not in [InstallationCount, RemoteInstallationCount, RemoteRealmCount]
-                and "realm" not in kwargs
-            ):
+            if table is not InstallationCount and "realm" not in kwargs:
                 if "user" in kwargs:
                     kwargs["realm"] = kwargs["user"].realm
                 elif "stream" in kwargs:
@@ -337,7 +283,7 @@ class TestProcessCountStat(AnalyticsTestCase):
         self.assertEqual(InstallationCount.objects.filter(property=stat.property).count(), 1)
 
         # clean stat, with update
-        current_time += self.HOUR
+        current_time = current_time + self.HOUR
         stat = self.make_dummy_count_stat("test stat")
         process_count_stat(stat, current_time)
         self.assertFillStateEquals(stat, current_time)
@@ -550,40 +496,58 @@ class TestCountStats(AnalyticsTestCase):
         # This huddle should not show up anywhere
         self.create_huddle_with_recipient()
 
-    def test_upload_quota_used_bytes(self) -> None:
-        stat = COUNT_STATS["upload_quota_used_bytes::day"]
+    def test_active_users_by_is_bot(self) -> None:
+        stat = COUNT_STATS["active_users:is_bot:day"]
         self.current_property = stat.property
 
-        user1 = self.create_user()
-        user2 = self.create_user()
-        user_second_realm = self.create_user(realm=self.second_realm)
+        # To be included
+        self.create_user(is_bot=True)
+        self.create_user(is_bot=True, date_joined=self.TIME_ZERO - 25 * self.HOUR)
+        self.create_user(is_bot=False)
 
-        self.create_attachment(user1, "file1", 100, self.TIME_LAST_HOUR, "text/plain")
-        attachment2 = self.create_attachment(user2, "file2", 200, self.TIME_LAST_HOUR, "text/plain")
-        self.create_attachment(user_second_realm, "file3", 10, self.TIME_LAST_HOUR, "text/plain")
+        # To be excluded
+        self.create_user(is_active=False)
 
         do_fill_count_stat_at_hour(stat, self.TIME_ZERO)
 
         self.assertTableState(
             RealmCount,
             ["value", "subgroup", "realm"],
-            [[300, None, self.default_realm], [10, None, self.second_realm]],
-        )
-
-        # Delete an attachment and run the CountStat job again the next day.
-        attachment2.delete()
-        do_fill_count_stat_at_hour(stat, self.TIME_ZERO + self.DAY)
-
-        self.assertTableState(
-            RealmCount,
-            ["value", "subgroup", "realm", "end_time"],
             [
-                [300, None, self.default_realm, self.TIME_ZERO],
-                [10, None, self.second_realm, self.TIME_ZERO],
-                [100, None, self.default_realm, self.TIME_ZERO + self.DAY],
-                [10, None, self.second_realm, self.TIME_ZERO + self.DAY],
+                [2, "true"],
+                [1, "false"],
+                [3, "false", self.second_realm],
+                [1, "false", self.no_message_realm],
             ],
         )
+        self.assertTableState(InstallationCount, ["value", "subgroup"], [[2, "true"], [5, "false"]])
+        self.assertTableState(UserCount, [], [])
+        self.assertTableState(StreamCount, [], [])
+
+    def test_active_users_by_is_bot_for_realm_constraint(self) -> None:
+        # For single Realm
+
+        COUNT_STATS = get_count_stats(self.default_realm)
+        stat = COUNT_STATS["active_users:is_bot:day"]
+        self.current_property = stat.property
+
+        # To be included
+        self.create_user(is_bot=True, date_joined=self.TIME_ZERO - 25 * self.HOUR)
+        self.create_user(is_bot=False)
+
+        # To be excluded
+        self.create_user(
+            email="test@second.analytics",
+            realm=self.second_realm,
+            date_joined=self.TIME_ZERO - 2 * self.DAY,
+        )
+
+        do_fill_count_stat_at_hour(stat, self.TIME_ZERO, self.default_realm)
+        self.assertTableState(RealmCount, ["value", "subgroup"], [[1, "true"], [1, "false"]])
+        # No aggregation to InstallationCount with realm constraint
+        self.assertTableState(InstallationCount, ["value", "subgroup"], [])
+        self.assertTableState(UserCount, [], [])
+        self.assertTableState(StreamCount, [], [])
 
     def test_messages_sent_by_is_bot(self) -> None:
         stat = COUNT_STATS["messages_sent:is_bot:hour"]
@@ -1338,11 +1302,6 @@ class TestDoIncrementLoggingStat(AnalyticsTestCase):
         do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
         self.assertTableState(RealmCount, ["value"], [[3]])
 
-    def test_do_increment_logging_start_query_count(self) -> None:
-        stat = LoggingCountStat("test", RealmCount, CountStat.DAY)
-        with self.assert_database_query_count(1):
-            do_increment_logging_stat(self.default_realm, stat, None, self.TIME_ZERO)
-
 
 class TestLoggingCountStats(AnalyticsTestCase):
     def test_aggregation(self) -> None:
@@ -1373,278 +1332,49 @@ class TestLoggingCountStats(AnalyticsTestCase):
         self.assertTableState(UserCount, ["property", "value"], [["user test", 1]])
         self.assertTableState(StreamCount, ["property", "value"], [["stream test", 1]])
 
-    @activate_push_notification_service()
-    def test_mobile_pushes_received_count(self) -> None:
-        self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe"
-        self.server = RemoteZulipServer.objects.create(
-            uuid=self.server_uuid,
-            api_key="magic_secret_api_key",
-            hostname="demo.example.com",
-            last_updated=timezone_now(),
+    def test_active_users_log_by_is_bot(self) -> None:
+        property = "active_users_log:is_bot:day"
+        user = do_create_user(
+            "email", "password", self.default_realm, "full_name", acting_user=None
         )
-
-        hamlet = self.example_user("hamlet")
-        token = "aaaa"
-
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.FCM,
-            token=hex_to_b64(token),
-            user_uuid=(hamlet.uuid),
-            server=self.server,
-        )
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.FCM,
-            token=hex_to_b64(token + "aa"),
-            user_uuid=(hamlet.uuid),
-            server=self.server,
-        )
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.APNS,
-            token=hex_to_b64(token),
-            user_uuid=str(hamlet.uuid),
-            server=self.server,
-        )
-
-        message = Message(
-            sender=hamlet,
-            recipient=self.example_user("othello").recipient,
-            realm_id=hamlet.realm_id,
-            content="This is test content",
-            rendered_content="This is test content",
-            date_sent=timezone_now(),
-            sending_client=get_client("test"),
-        )
-        message.set_topic_name("Test topic")
-        message.save()
-        gcm_payload, gcm_options = get_message_payload_gcm(hamlet, message)
-        apns_payload = get_message_payload_apns(
-            hamlet, message, NotificationTriggers.DIRECT_MESSAGE
-        )
-
-        # First we'll make a request without providing realm_uuid. That means
-        # the bouncer can't increment the RemoteRealmCount stat, and only
-        # RemoteInstallationCount will be incremented.
-        payload = {
-            "user_id": hamlet.id,
-            "user_uuid": str(hamlet.uuid),
-            "gcm_payload": gcm_payload,
-            "apns_payload": apns_payload,
-            "gcm_options": gcm_options,
-        }
-        now = timezone_now()
-        with (
-            time_machine.travel(now, tick=False),
-            mock.patch("zilencer.views.send_android_push_notification", return_value=1),
-            mock.patch("zilencer.views.send_apple_push_notification", return_value=1),
-            mock.patch(
-                "corporate.lib.stripe.RemoteServerBillingSession.current_count_for_billed_licenses",
-                return_value=10,
-            ),
-            self.assertLogs("zilencer.views", level="INFO"),
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # There are 3 devices we created for the user:
-        # 1. The mobile_pushes_received increment should match that number.
-        # 2. mobile_pushes_forwarded only counts successful deliveries, and we've set up
-        #    the mocks above to simulate 1 successful android and 1 successful apple delivery.
-        #    Thus the increment should be just 2.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    3,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    2,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
+        self.assertEqual(
+            1,
+            RealmCount.objects.filter(property=property, subgroup=False).aggregate(Sum("value"))[
+                "value__sum"
             ],
         )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_received::day").exists()
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_forwarded::day").exists()
-        )
-
-        # Now provide the realm_uuid. However, the RemoteRealm record doesn't exist yet, so it'll
-        # still be ignored.
-        payload = {
-            "user_id": hamlet.id,
-            "user_uuid": str(hamlet.uuid),
-            "realm_uuid": str(hamlet.realm.uuid),
-            "gcm_payload": gcm_payload,
-            "apns_payload": apns_payload,
-            "gcm_options": gcm_options,
-        }
-        with (
-            time_machine.travel(now, tick=False),
-            mock.patch("zilencer.views.send_android_push_notification", return_value=1),
-            mock.patch("zilencer.views.send_apple_push_notification", return_value=1),
-            mock.patch(
-                "corporate.lib.stripe.RemoteServerBillingSession.current_count_for_billed_licenses",
-                return_value=10,
-            ),
-            self.assertLogs("zilencer.views", level="INFO"),
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # The RemoteInstallationCount records get incremented again, but the RemoteRealmCount
-        # remains ignored due to missing RemoteRealm record.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    6,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    4,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
+        do_deactivate_user(user, acting_user=None)
+        self.assertEqual(
+            0,
+            RealmCount.objects.filter(property=property, subgroup=False).aggregate(Sum("value"))[
+                "value__sum"
             ],
         )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_received::day").exists()
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_forwarded::day").exists()
-        )
-
-        # Create the RemoteRealm registration and repeat the above. This time RemoteRealmCount
-        # stats should be collected.
-        realm = hamlet.realm
-        remote_realm = RemoteRealm.objects.create(
-            server=self.server,
-            uuid=realm.uuid,
-            uuid_owner_secret=realm.uuid_owner_secret,
-            host=realm.host,
-            realm_deactivated=realm.deactivated,
-            realm_date_created=realm.date_created,
-        )
-
-        with (
-            time_machine.travel(now, tick=False),
-            mock.patch("zilencer.views.send_android_push_notification", return_value=1),
-            mock.patch("zilencer.views.send_apple_push_notification", return_value=1),
-            mock.patch(
-                "corporate.lib.stripe.RemoteRealmBillingSession.current_count_for_billed_licenses",
-                return_value=10,
-            ),
-            self.assertLogs("zilencer.views", level="INFO"),
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # The RemoteInstallationCount records get incremented again, and the RemoteRealmCount
-        # gets collected.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    9,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    6,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
+        do_activate_mirror_dummy_user(user, acting_user=None)
+        self.assertEqual(
+            1,
+            RealmCount.objects.filter(property=property, subgroup=False).aggregate(Sum("value"))[
+                "value__sum"
             ],
         )
-        self.assertTableState(
-            RemoteRealmCount,
-            ["property", "value", "subgroup", "server", "remote_realm", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    3,
-                    None,
-                    self.server,
-                    remote_realm,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    2,
-                    None,
-                    self.server,
-                    remote_realm,
-                    None,
-                    ceiling_to_day(now),
-                ],
+        do_deactivate_user(user, acting_user=None)
+        self.assertEqual(
+            0,
+            RealmCount.objects.filter(property=property, subgroup=False).aggregate(Sum("value"))[
+                "value__sum"
+            ],
+        )
+        do_reactivate_user(user, acting_user=None)
+        self.assertEqual(
+            1,
+            RealmCount.objects.filter(property=property, subgroup=False).aggregate(Sum("value"))[
+                "value__sum"
             ],
         )
 
     def test_invites_sent(self) -> None:
         property = "invites_sent::day"
 
-        @contextmanager
-        def invite_context(
-            too_many_recent_realm_invites: bool = False, failure: bool = False
-        ) -> Iterator[None]:
-            managers: list[AbstractContextManager[Any]] = [
-                mock.patch(
-                    "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
-                ),
-                self.captureOnCommitCallbacks(execute=True),
-            ]
-            if failure:
-                managers.append(self.assertRaises(InvitationError))
-            with ExitStack() as stack:
-                for mgr in managers:
-                    stack.enter_context(mgr)
-                yield
-
         def assertInviteCountEquals(count: int) -> None:
             self.assertEqual(
                 count,
@@ -1657,49 +1387,48 @@ class TestLoggingCountStats(AnalyticsTestCase):
         stream, _ = self.create_stream_with_recipient()
 
         invite_expires_in_minutes = 2 * 24 * 60
-        with invite_context():
+        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
             do_invite_users(
                 user,
                 ["user1@domain.tld", "user2@domain.tld"],
                 [stream],
-                include_realm_default_subscriptions=False,
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
         assertInviteCountEquals(2)
 
         # We currently send emails when re-inviting users that haven't
         # turned into accounts, so count them towards the total
-        with invite_context():
+        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
             do_invite_users(
                 user,
                 ["user1@domain.tld", "user2@domain.tld"],
                 [stream],
-                include_realm_default_subscriptions=False,
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
         assertInviteCountEquals(4)
 
         # Test mix of good and malformed invite emails
-        with invite_context(failure=True):
+        with self.assertRaises(InvitationError), mock.patch(
+            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
+        ):
             do_invite_users(
                 user,
                 ["user3@domain.tld", "malformed"],
                 [stream],
-                include_realm_default_subscriptions=False,
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
         assertInviteCountEquals(4)
 
         # Test inviting existing users
-        with invite_context():
-            skipped = do_invite_users(
+        with self.assertRaises(InvitationError), mock.patch(
+            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
+        ):
+            do_invite_users(
                 user,
                 ["first@domain.tld", "user4@domain.tld"],
                 [stream],
-                include_realm_default_subscriptions=False,
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
-            self.assert_length(skipped, 1)
         assertInviteCountEquals(5)
 
         # Revoking invite should not give you credit
@@ -1709,8 +1438,8 @@ class TestLoggingCountStats(AnalyticsTestCase):
         assertInviteCountEquals(5)
 
         # Resending invite should cost you
-        with invite_context():
-            do_send_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
+        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
+            do_resend_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
         assertInviteCountEquals(6)
 
     def test_messages_read_hour(self) -> None:
@@ -1820,12 +1549,12 @@ class TestActiveUsersAudit(AnalyticsTestCase):
     @override
     def setUp(self) -> None:
         super().setUp()
-        self.user = self.create_user(skip_auditlog=True)
+        self.user = self.create_user()
         self.stat = COUNT_STATS["active_users_audit:is_bot:day"]
         self.current_property = self.stat.property
 
     def add_event(
-        self, event_type: int, days_offset: float, user: UserProfile | None = None
+        self, event_type: int, days_offset: float, user: Optional[UserProfile] = None
     ) -> None:
         hours_offset = int(24 * days_offset)
         if user is None:
@@ -1841,25 +1570,25 @@ class TestActiveUsersAudit(AnalyticsTestCase):
         self.add_event(RealmAuditLog.USER_CREATED, 1)
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 0)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["subgroup"], [["false"]])
+        self.assertTableState(UserCount, ["subgroup"], [["false"]])
 
     def test_user_reactivated_in_future(self) -> None:
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 1)
         self.add_event(RealmAuditLog.USER_REACTIVATED, 0)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, [], [])
+        self.assertTableState(UserCount, [], [])
 
     def test_user_active_then_deactivated_same_day(self) -> None:
         self.add_event(RealmAuditLog.USER_CREATED, 1)
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 0.5)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, [], [])
+        self.assertTableState(UserCount, [], [])
 
     def test_user_unactive_then_activated_same_day(self) -> None:
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 1)
         self.add_event(RealmAuditLog.USER_REACTIVATED, 0.5)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["subgroup"], [["false"]])
+        self.assertTableState(UserCount, ["subgroup"], [["false"]])
 
     # Arguably these next two tests are duplicates of the _in_future tests, but are
     # a guard against future refactorings where they may no longer be duplicates
@@ -1868,14 +1597,14 @@ class TestActiveUsersAudit(AnalyticsTestCase):
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 1)
         process_count_stat(self.stat, self.TIME_ZERO)
         self.assertTableState(
-            RealmCount, ["subgroup", "end_time"], [["false", self.TIME_ZERO - self.DAY]]
+            UserCount, ["subgroup", "end_time"], [["false", self.TIME_ZERO - self.DAY]]
         )
 
     def test_user_deactivated_then_reactivated_with_day_gap(self) -> None:
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 2)
         self.add_event(RealmAuditLog.USER_REACTIVATED, 1)
         process_count_stat(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["subgroup"], [["false"]])
+        self.assertTableState(UserCount, ["subgroup"], [["false"]])
 
     def test_event_types(self) -> None:
         self.add_event(RealmAuditLog.USER_CREATED, 4)
@@ -1885,7 +1614,7 @@ class TestActiveUsersAudit(AnalyticsTestCase):
         for i in range(4):
             do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO - i * self.DAY)
         self.assertTableState(
-            RealmCount,
+            UserCount,
             ["subgroup", "end_time"],
             [["false", self.TIME_ZERO - i * self.DAY] for i in [3, 1, 0]],
         )
@@ -1893,14 +1622,19 @@ class TestActiveUsersAudit(AnalyticsTestCase):
     # Also tests that aggregation to RealmCount and InstallationCount is
     # being done, and that we're storing the user correctly in UserCount
     def test_multiple_users_realms_and_bots(self) -> None:
-        user1 = self.create_user(skip_auditlog=True)
-        user2 = self.create_user(skip_auditlog=True)
+        user1 = self.create_user()
+        user2 = self.create_user()
         second_realm = do_create_realm(string_id="moo", name="moo")
-        user3 = self.create_user(skip_auditlog=True, realm=second_realm)
-        user4 = self.create_user(skip_auditlog=True, realm=second_realm, is_bot=True)
+        user3 = self.create_user(realm=second_realm)
+        user4 = self.create_user(realm=second_realm, is_bot=True)
         for user in [user1, user2, user3, user4]:
             self.add_event(RealmAuditLog.USER_CREATED, 1, user=user)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
+        self.assertTableState(
+            UserCount,
+            ["subgroup", "user"],
+            [["false", user1], ["false", user2], ["false", user3], ["true", user4]],
+        )
         self.assertTableState(
             RealmCount,
             ["value", "subgroup", "realm"],
@@ -1924,7 +1658,7 @@ class TestActiveUsersAudit(AnalyticsTestCase):
         self.add_event(RealmAuditLog.USER_CREATED, 2)
         process_count_stat(self.stat, self.TIME_ZERO)
         self.assertTableState(
-            RealmCount,
+            UserCount,
             ["subgroup", "end_time"],
             [["false", self.TIME_ZERO], ["false", self.TIME_ZERO - self.DAY]],
         )
@@ -1934,37 +1668,39 @@ class TestActiveUsersAudit(AnalyticsTestCase):
     # that situation doesn't throw an error.
     def test_empty_realm_or_user_with_no_relevant_activity(self) -> None:
         self.add_event(RealmAuditLog.USER_SOFT_ACTIVATED, 1)
-        self.create_user(skip_auditlog=True)  # also test a user with no RealmAuditLog entries
+        self.create_user()  # also test a user with no RealmAuditLog entries
         do_create_realm(string_id="moo", name="moo")
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, [], [])
+        self.assertTableState(UserCount, [], [])
 
     def test_max_audit_entry_is_unrelated(self) -> None:
         self.add_event(RealmAuditLog.USER_CREATED, 1)
         self.add_event(RealmAuditLog.USER_SOFT_ACTIVATED, 0.5)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["subgroup"], [["false"]])
+        self.assertTableState(UserCount, ["subgroup"], [["false"]])
 
     # Simultaneous related audit entries should not be allowed, and so not testing for that.
     def test_simultaneous_unrelated_audit_entry(self) -> None:
         self.add_event(RealmAuditLog.USER_CREATED, 1)
         self.add_event(RealmAuditLog.USER_SOFT_ACTIVATED, 1)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["subgroup"], [["false"]])
+        self.assertTableState(UserCount, ["subgroup"], [["false"]])
 
     def test_simultaneous_max_audit_entries_of_different_users(self) -> None:
-        user1 = self.create_user(skip_auditlog=True)
-        user2 = self.create_user(skip_auditlog=True)
-        user3 = self.create_user(skip_auditlog=True)
+        user1 = self.create_user()
+        user2 = self.create_user()
+        user3 = self.create_user()
         self.add_event(RealmAuditLog.USER_CREATED, 0.5, user=user1)
         self.add_event(RealmAuditLog.USER_CREATED, 0.5, user=user2)
         self.add_event(RealmAuditLog.USER_CREATED, 1, user=user3)
         self.add_event(RealmAuditLog.USER_DEACTIVATED, 0.5, user=user3)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
-        self.assertTableState(RealmCount, ["value", "subgroup"], [[2, "false"]])
+        self.assertTableState(UserCount, ["user", "subgroup"], [[user1, "false"], [user2, "false"]])
 
     def test_end_to_end_with_actions_dot_py(self) -> None:
-        do_create_user("email1", "password", self.default_realm, "full_name", acting_user=None)
+        user1 = do_create_user(
+            "email1", "password", self.default_realm, "full_name", acting_user=None
+        )
         user2 = do_create_user(
             "email2", "password", self.default_realm, "full_name", acting_user=None
         )
@@ -1979,15 +1715,17 @@ class TestActiveUsersAudit(AnalyticsTestCase):
         do_reactivate_user(user4, acting_user=None)
         end_time = floor_to_day(timezone_now()) + self.DAY
         do_fill_count_stat_at_hour(self.stat, end_time)
-        self.assertTrue(
-            RealmCount.objects.filter(
-                realm=self.default_realm,
-                property=self.current_property,
-                subgroup="false",
-                end_time=end_time,
-                value=3,
-            ).exists()
-        )
+        for user in [user1, user3, user4]:
+            self.assertTrue(
+                UserCount.objects.filter(
+                    user=user,
+                    property=self.current_property,
+                    subgroup="false",
+                    end_time=end_time,
+                    value=1,
+                ).exists()
+            )
+        self.assertFalse(UserCount.objects.filter(user=user2, end_time=end_time).exists())
 
 
 class TestRealmActiveHumans(AnalyticsTestCase):
@@ -1997,42 +1735,57 @@ class TestRealmActiveHumans(AnalyticsTestCase):
         self.stat = COUNT_STATS["realm_active_humans::day"]
         self.current_property = self.stat.property
 
-    def mark_15day_active(self, user: UserProfile, end_time: datetime | None = None) -> None:
+    def mark_audit_active(self, user: UserProfile, end_time: Optional[datetime] = None) -> None:
+        if end_time is None:
+            end_time = self.TIME_ZERO
+        UserCount.objects.create(
+            user=user,
+            realm=user.realm,
+            property="active_users_audit:is_bot:day",
+            subgroup=orjson.dumps(user.is_bot).decode(),
+            end_time=end_time,
+            value=1,
+        )
+
+    def mark_15day_active(self, user: UserProfile, end_time: Optional[datetime] = None) -> None:
         if end_time is None:
             end_time = self.TIME_ZERO
         UserCount.objects.create(
             user=user, realm=user.realm, property="15day_actives::day", end_time=end_time, value=1
         )
 
-    def test_basic_logic(self) -> None:
+    def test_basic_boolean_logic(self) -> None:
         user = self.create_user()
+        self.mark_audit_active(user, end_time=self.TIME_ZERO - self.DAY)
         self.mark_15day_active(user, end_time=self.TIME_ZERO)
+        self.mark_audit_active(user, end_time=self.TIME_ZERO + self.DAY)
         self.mark_15day_active(user, end_time=self.TIME_ZERO + self.DAY)
 
         for i in [-1, 0, 1]:
             do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i * self.DAY)
-        self.assertTableState(
-            RealmCount, ["value", "end_time"], [[1, self.TIME_ZERO], [1, self.TIME_ZERO + self.DAY]]
-        )
+        self.assertTableState(RealmCount, ["value", "end_time"], [[1, self.TIME_ZERO + self.DAY]])
 
     def test_bots_not_counted(self) -> None:
         bot = self.create_user(is_bot=True)
+        self.mark_audit_active(bot)
         self.mark_15day_active(bot)
         do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO)
         self.assertTableState(RealmCount, [], [])
 
     def test_multiple_users_realms_and_times(self) -> None:
-        user1 = self.create_user(date_joined=self.TIME_ZERO - 2 * self.DAY)
-        user2 = self.create_user(date_joined=self.TIME_ZERO - 2 * self.DAY)
+        user1 = self.create_user()
+        user2 = self.create_user()
         second_realm = do_create_realm(string_id="second", name="second")
-        user3 = self.create_user(date_joined=self.TIME_ZERO - 2 * self.DAY, realm=second_realm)
-        user4 = self.create_user(date_joined=self.TIME_ZERO - 2 * self.DAY, realm=second_realm)
-        user5 = self.create_user(date_joined=self.TIME_ZERO - 2 * self.DAY, realm=second_realm)
+        user3 = self.create_user(realm=second_realm)
+        user4 = self.create_user(realm=second_realm)
+        user5 = self.create_user(realm=second_realm)
 
-        for user in [user1, user3, user4]:
-            self.mark_15day_active(user, end_time=self.TIME_ZERO - self.DAY)
         for user in [user1, user2, user3, user4, user5]:
+            self.mark_audit_active(user)
             self.mark_15day_active(user)
+        for user in [user1, user3, user4]:
+            self.mark_audit_active(user, end_time=self.TIME_ZERO - self.DAY)
+            self.mark_15day_active(user, end_time=self.TIME_ZERO - self.DAY)
 
         for i in [-1, 0, 1]:
             do_fill_count_stat_at_hour(self.stat, self.TIME_ZERO + i * self.DAY)
@@ -2040,14 +1793,17 @@ class TestRealmActiveHumans(AnalyticsTestCase):
             RealmCount,
             ["value", "realm", "end_time"],
             [
-                [1, self.default_realm, self.TIME_ZERO - self.DAY],
-                [2, second_realm, self.TIME_ZERO - self.DAY],
                 [2, self.default_realm, self.TIME_ZERO],
                 [3, second_realm, self.TIME_ZERO],
+                [1, self.default_realm, self.TIME_ZERO - self.DAY],
+                [2, second_realm, self.TIME_ZERO - self.DAY],
             ],
         )
 
         # Check that adding spurious entries doesn't make a difference
+        self.mark_audit_active(user1, end_time=self.TIME_ZERO + self.DAY)
+        self.mark_15day_active(user2, end_time=self.TIME_ZERO + self.DAY)
+        self.mark_15day_active(user2, end_time=self.TIME_ZERO - self.DAY)
         self.create_user()
         third_realm = do_create_realm(string_id="third", name="third")
         self.create_user(realm=third_realm)
@@ -2060,10 +1816,10 @@ class TestRealmActiveHumans(AnalyticsTestCase):
             RealmCount,
             ["value", "realm", "end_time"],
             [
-                [1, self.default_realm, self.TIME_ZERO - self.DAY],
-                [2, second_realm, self.TIME_ZERO - self.DAY],
                 [2, self.default_realm, self.TIME_ZERO],
                 [3, second_realm, self.TIME_ZERO],
+                [1, self.default_realm, self.TIME_ZERO - self.DAY],
+                [2, second_realm, self.TIME_ZERO - self.DAY],
             ],
         )
 
@@ -2093,26 +1849,3 @@ class TestRealmActiveHumans(AnalyticsTestCase):
             1,
         )
         self.assertEqual(RealmCount.objects.filter(property="realm_active_humans::day").count(), 1)
-
-
-class GetLastIdFromServerTest(ZulipTestCase):
-    def test_get_last_id_from_server_ignores_null(self) -> None:
-        """
-        Verifies that get_last_id_from_server ignores null remote_ids, since this goes
-        against the default Postgres ordering behavior, which treats nulls as the largest value.
-        """
-        self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe"
-        self.server = RemoteZulipServer.objects.create(
-            uuid=self.server_uuid,
-            api_key="magic_secret_api_key",
-            hostname="demo.example.com",
-            last_updated=timezone_now(),
-        )
-        first = RemoteInstallationCount.objects.create(
-            end_time=timezone_now(), server=self.server, property="test", value=1, remote_id=1
-        )
-        RemoteInstallationCount.objects.create(
-            end_time=timezone_now(), server=self.server, property="test2", value=1, remote_id=None
-        )
-        result = get_last_id_from_server(self.server, RemoteInstallationCount)
-        self.assertEqual(result, first.remote_id)

analytics/tests/test_stats_views.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+from typing import List, Optional
 
 from django.utils.timezone import now as timezone_now
 from typing_extensions import override
@@ -9,8 +10,7 @@ from analytics.models import FillState, RealmCount, StreamCount, UserCount
 from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client
-from zerver.models.realms import get_realm
+from zerver.models import Client, get_realm
 
 
 class TestStatsEndpoint(ZulipTestCase):
@@ -83,11 +83,11 @@ class TestGetChartData(ZulipTestCase):
             ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
         ]
 
-    def data(self, i: int) -> list[int]:
+    def data(self, i: int) -> List[int]:
         return [0, 0, i, 0]
 
     def insert_data(
-        self, stat: CountStat, realm_subgroups: list[str | None], user_subgroups: list[str]
+        self, stat: CountStat, realm_subgroups: List[Optional[str]], user_subgroups: List[str]
     ) -> None:
         if stat.frequency == CountStat.HOUR:
             insert_time = self.end_times_hour[2]
@@ -191,21 +191,21 @@ class TestGetChartData(ZulipTestCase):
                 "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
                 "frequency": CountStat.DAY,
                 "everyone": {
-                    "Public channels": self.data(100),
-                    "Private channels": self.data(0),
+                    "Public streams": self.data(100),
+                    "Private streams": self.data(0),
                     "Direct messages": self.data(101),
                     "Group direct messages": self.data(0),
                 },
                 "user": {
-                    "Public channels": self.data(200),
-                    "Private channels": self.data(201),
+                    "Public streams": self.data(200),
+                    "Private streams": self.data(201),
                     "Direct messages": self.data(0),
                     "Group direct messages": self.data(0),
                 },
                 "display_order": [
                     "Direct messages",
-                    "Public channels",
-                    "Private channels",
+                    "Public streams",
+                    "Private streams",
                     "Group direct messages",
                 ],
                 "result": "success",
@@ -305,7 +305,7 @@ class TestGetChartData(ZulipTestCase):
             },
             subdomain="zephyr",
         )
-        self.assert_json_error(result, "Invalid channel ID")
+        self.assert_json_error(result, "Invalid stream ID")
 
     def test_include_empty_subgroups(self) -> None:
         FillState.objects.create(
@@ -342,8 +342,8 @@ class TestGetChartData(ZulipTestCase):
         self.assertEqual(
             data["everyone"],
             {
-                "Public channels": [0],
-                "Private channels": [0],
+                "Public streams": [0],
+                "Private streams": [0],
                 "Direct messages": [0],
                 "Group direct messages": [0],
             },
@@ -351,8 +351,8 @@ class TestGetChartData(ZulipTestCase):
         self.assertEqual(
             data["user"],
             {
-                "Public channels": [0],
-                "Private channels": [0],
+                "Public streams": [0],
+                "Private streams": [0],
                 "Direct messages": [0],
                 "Group direct messages": [0],
             },
@@ -604,7 +604,7 @@ class TestGetChartData(ZulipTestCase):
 
 class TestGetChartDataHelpers(ZulipTestCase):
     def test_sort_by_totals(self) -> None:
-        empty: list[int] = []
+        empty: List[int] = []
         value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
         self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
 
@@ -660,9 +660,7 @@ class TestMapArrays(ZulipTestCase):
             "website": [1, 2, 3],
             "ZulipiOS": [1, 2, 3],
             "ZulipElectron": [2, 5, 7],
-            "ZulipMobile": [1, 2, 3],
-            "ZulipMobile/flutter": [1, 1, 1],
-            "ZulipFlutter": [1, 1, 1],
+            "ZulipMobile": [1, 5, 7],
             "ZulipPython": [1, 2, 3],
             "API: Python": [1, 2, 3],
             "SomethingRandom": [4, 5, 6],
@@ -677,8 +675,7 @@ class TestMapArrays(ZulipTestCase):
                 "Old desktop app": [32, 36, 39],
                 "Old iOS app": [1, 2, 3],
                 "Desktop app": [2, 5, 7],
-                "Mobile app (React Native)": [1, 2, 3],
-                "Mobile app beta (Flutter)": [2, 2, 2],
+                "Mobile app": [1, 5, 7],
                 "Web app": [1, 2, 3],
                 "Python API": [2, 4, 6],
                 "SomethingRandom": [4, 5, 6],

analytics/tests/test_support_views.py

@@ -0,0 +1,791 @@
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Optional
+from unittest import mock
+
+import orjson
+from django.utils.timezone import now as timezone_now
+from typing_extensions import override
+
+from corporate.lib.stripe import add_months, update_sponsorship_status
+from corporate.models import Customer, CustomerPlan, LicenseLedger, get_customer_by_realm
+from zerver.actions.invites import do_create_multiuse_invite_link
+from zerver.actions.realm_settings import do_change_realm_org_type, do_send_realm_reactivation_email
+from zerver.actions.user_settings import do_change_user_setting
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm
+from zerver.models import (
+    MultiuseInvite,
+    PreregistrationUser,
+    Realm,
+    UserMessage,
+    UserProfile,
+    get_org_type_display_name,
+    get_realm,
+)
+
+if TYPE_CHECKING:
+    from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
+
+import uuid
+
+from zilencer.models import RemoteZulipServer
+
+
+class TestRemoteServerSupportEndpoint(ZulipTestCase):
+    @override
+    def setUp(self) -> None:
+        super().setUp()
+
+        # Set up some initial example data.
+        for i in range(20):
+            hostname = f"zulip-{i}.example.com"
+            RemoteZulipServer.objects.create(
+                hostname=hostname, contact_email=f"admin@{hostname}", plan_type=1, uuid=uuid.uuid4()
+            )
+
+    def test_search(self) -> None:
+        self.login("cordelia")
+
+        result = self.client_get("/activity/remote/support")
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        # Iago is the user with the appropriate permissions to access this page.
+        self.login("iago")
+        assert self.example_user("iago").is_staff
+
+        result = self.client_get("/activity/remote/support")
+        self.assert_in_success_response(
+            [
+                'input type="text" name="q" class="input-xxlarge search-query" placeholder="hostname or contact email"'
+            ],
+            result,
+        )
+
+        result = self.client_get("/activity/remote/support", {"q": "zulip-1.example.com"})
+        self.assert_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
+        self.assert_not_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
+
+        result = self.client_get("/activity/remote/support", {"q": "example.com"})
+        for i in range(20):
+            self.assert_in_success_response([f"<h3>zulip-{i}.example.com</h3>"], result)
+
+        result = self.client_get("/activity/remote/support", {"q": "admin@zulip-2.example.com"})
+        self.assert_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
+        self.assert_in_success_response(["<b>Contact email</b>: admin@zulip-2.example.com"], result)
+        self.assert_not_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
+
+
+class TestSupportEndpoint(ZulipTestCase):
+    def test_search(self) -> None:
+        reset_email_visibility_to_everyone_in_zulip_realm()
+        lear_user = self.lear_user("king")
+        lear_user.is_staff = True
+        lear_user.save(update_fields=["is_staff"])
+        lear_realm = get_realm("lear")
+
+        def assert_user_details_in_html_response(
+            html_response: "TestHttpResponse", full_name: str, email: str, role: str
+        ) -> None:
+            self.assert_in_success_response(
+                [
+                    '<span class="label">user</span>\n',
+                    f"<h3>{full_name}</h3>",
+                    f"<b>Email</b>: {email}",
+                    "<b>Is active</b>: True<br />",
+                    f"<b>Role</b>: {role}<br />",
+                ],
+                html_response,
+            )
+
+        def create_invitation(
+            stream: str, invitee_email: str, realm: Optional[Realm] = None
+        ) -> None:
+            invite_expires_in_minutes = 10 * 24 * 60
+            self.client_post(
+                "/json/invites",
+                {
+                    "invitee_emails": [invitee_email],
+                    "stream_ids": orjson.dumps([self.get_stream_id(stream, realm)]).decode(),
+                    "invite_expires_in_minutes": invite_expires_in_minutes,
+                    "invite_as": PreregistrationUser.INVITE_AS["MEMBER"],
+                },
+                subdomain=realm.string_id if realm is not None else "zulip",
+            )
+
+        def check_hamlet_user_query_result(result: "TestHttpResponse") -> None:
+            assert_user_details_in_html_response(
+                result, "King Hamlet", self.example_email("hamlet"), "Member"
+            )
+            self.assert_in_success_response(
+                [
+                    f"<b>Admins</b>: {self.example_email('iago')}\n",
+                    f"<b>Owners</b>: {self.example_email('desdemona')}\n",
+                    'class="copy-button" data-copytext="{}">'.format(self.example_email("iago")),
+                    'class="copy-button" data-copytext="{}">'.format(
+                        self.example_email("desdemona")
+                    ),
+                ],
+                result,
+            )
+
+        def check_lear_user_query_result(result: "TestHttpResponse") -> None:
+            assert_user_details_in_html_response(
+                result, lear_user.full_name, lear_user.email, "Member"
+            )
+
+        def check_othello_user_query_result(result: "TestHttpResponse") -> None:
+            assert_user_details_in_html_response(
+                result, "Othello, the Moor of Venice", self.example_email("othello"), "Member"
+            )
+
+        def check_polonius_user_query_result(result: "TestHttpResponse") -> None:
+            assert_user_details_in_html_response(
+                result, "Polonius", self.example_email("polonius"), "Guest"
+            )
+
+        def check_zulip_realm_query_result(result: "TestHttpResponse") -> None:
+            zulip_realm = get_realm("zulip")
+            first_human_user = zulip_realm.get_first_human_user()
+            assert first_human_user is not None
+            self.assert_in_success_response(
+                [
+                    f"<b>First human user</b>: {first_human_user.delivery_email}\n",
+                    f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"',
+                    "Zulip Dev</h3>",
+                    '<option value="1" selected>Self-hosted</option>',
+                    '<option value="2" >Limited</option>',
+                    'input type="number" name="discount" value="None"',
+                    '<option value="active" selected>Active</option>',
+                    '<option value="deactivated" >Deactivated</option>',
+                    f'<option value="{zulip_realm.org_type}" selected>',
+                    'scrub-realm-button">',
+                    'data-string-id="zulip"',
+                ],
+                result,
+            )
+
+        def check_lear_realm_query_result(result: "TestHttpResponse") -> None:
+            self.assert_in_success_response(
+                [
+                    f'<input type="hidden" name="realm_id" value="{lear_realm.id}"',
+                    "Lear &amp; Co.</h3>",
+                    '<option value="1" selected>Self-hosted</option>',
+                    '<option value="2" >Limited</option>',
+                    'input type="number" name="discount" value="None"',
+                    '<option value="active" selected>Active</option>',
+                    '<option value="deactivated" >Deactivated</option>',
+                    'scrub-realm-button">',
+                    'data-string-id="lear"',
+                    "<b>Name</b>: Zulip Cloud Standard",
+                    "<b>Status</b>: Active",
+                    "<b>Billing schedule</b>: Annual",
+                    "<b>Licenses</b>: 2/10 (Manual)",
+                    "<b>Price per license</b>: $80.0",
+                    "<b>Next invoice date</b>: 02 January 2017",
+                    '<option value="send_invoice" selected>',
+                    '<option value="charge_automatically" >',
+                ],
+                result,
+            )
+
+        def check_preregistration_user_query_result(
+            result: "TestHttpResponse", email: str, invite: bool = False
+        ) -> None:
+            self.assert_in_success_response(
+                [
+                    '<span class="label">preregistration user</span>\n',
+                    f"<b>Email</b>: {email}",
+                ],
+                result,
+            )
+            if invite:
+                self.assert_in_success_response(['<span class="label">invite</span>'], result)
+                self.assert_in_success_response(
+                    [
+                        "<b>Expires in</b>: 1\xa0week, 3\xa0days",
+                        "<b>Status</b>: Link has not been used",
+                    ],
+                    result,
+                )
+                self.assert_in_success_response([], result)
+            else:
+                self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
+                self.assert_in_success_response(
+                    [
+                        "<b>Expires in</b>: 1\xa0day",
+                        "<b>Status</b>: Link has not been used",
+                    ],
+                    result,
+                )
+
+        def check_realm_creation_query_result(result: "TestHttpResponse", email: str) -> None:
+            self.assert_in_success_response(
+                [
+                    '<span class="label">preregistration user</span>\n',
+                    '<span class="label">realm creation</span>\n',
+                    "<b>Link</b>: http://testserver/accounts/do_confirm/",
+                    "<b>Expires in</b>: 1\xa0day",
+                ],
+                result,
+            )
+
+        def check_multiuse_invite_link_query_result(result: "TestHttpResponse") -> None:
+            self.assert_in_success_response(
+                [
+                    '<span class="label">multiuse invite</span>\n',
+                    "<b>Link</b>: http://zulip.testserver/join/",
+                    "<b>Expires in</b>: 1\xa0week, 3\xa0days",
+                ],
+                result,
+            )
+
+        def check_realm_reactivation_link_query_result(result: "TestHttpResponse") -> None:
+            self.assert_in_success_response(
+                [
+                    '<span class="label">realm reactivation</span>\n',
+                    "<b>Link</b>: http://zulip.testserver/reactivate/",
+                    "<b>Expires in</b>: 1\xa0day",
+                ],
+                result,
+            )
+
+        def get_check_query_result(
+            query: str, count: int, subdomain: str = "zulip"
+        ) -> "TestHttpResponse":
+            result = self.client_get("/activity/support", {"q": query}, subdomain=subdomain)
+            self.assertEqual(result.content.decode().count("support-query-result"), count)
+            return result
+
+        self.login("cordelia")
+
+        result = self.client_get("/activity/support")
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        self.login("iago")
+
+        do_change_user_setting(
+            self.example_user("hamlet"),
+            "email_address_visibility",
+            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+            acting_user=None,
+        )
+
+        customer = Customer.objects.create(realm=lear_realm, stripe_customer_id="cus_123")
+        now = datetime(2016, 1, 2, tzinfo=timezone.utc)
+        plan = CustomerPlan.objects.create(
+            customer=customer,
+            billing_cycle_anchor=now,
+            billing_schedule=CustomerPlan.ANNUAL,
+            tier=CustomerPlan.STANDARD,
+            price_per_license=8000,
+            next_invoice_date=add_months(now, 12),
+        )
+        LicenseLedger.objects.create(
+            licenses=10,
+            licenses_at_next_renewal=10,
+            event_time=timezone_now(),
+            is_renewal=True,
+            plan=plan,
+        )
+
+        result = self.client_get("/activity/support")
+        self.assert_in_success_response(
+            ['<input type="text" name="q" class="input-xxlarge search-query"'], result
+        )
+
+        result = get_check_query_result(self.example_email("hamlet"), 1)
+        check_hamlet_user_query_result(result)
+        check_zulip_realm_query_result(result)
+
+        # Search should be case-insensitive:
+        assert self.example_email("hamlet") != self.example_email("hamlet").upper()
+        result = get_check_query_result(self.example_email("hamlet").upper(), 1)
+        check_hamlet_user_query_result(result)
+        check_zulip_realm_query_result(result)
+
+        result = get_check_query_result(lear_user.email, 1)
+        check_lear_user_query_result(result)
+        check_lear_realm_query_result(result)
+
+        result = get_check_query_result(self.example_email("polonius"), 1)
+        check_polonius_user_query_result(result)
+        check_zulip_realm_query_result(result)
+
+        result = get_check_query_result("lear", 1)
+        check_lear_realm_query_result(result)
+
+        result = get_check_query_result("http://lear.testserver", 1)
+        check_lear_realm_query_result(result)
+
+        with self.settings(REALM_HOSTS={"zulip": "localhost"}):
+            result = get_check_query_result("http://localhost", 1)
+            check_zulip_realm_query_result(result)
+
+        result = get_check_query_result("hamlet@zulip.com, lear", 2)
+        check_hamlet_user_query_result(result)
+        check_zulip_realm_query_result(result)
+        check_lear_realm_query_result(result)
+
+        result = get_check_query_result("King hamlet,lear", 2)
+        check_hamlet_user_query_result(result)
+        check_zulip_realm_query_result(result)
+        check_lear_realm_query_result(result)
+
+        result = get_check_query_result("Othello, the Moor of Venice", 1)
+        check_othello_user_query_result(result)
+        check_zulip_realm_query_result(result)
+
+        result = get_check_query_result("lear, Hamlet <hamlet@zulip.com>", 2)
+        check_hamlet_user_query_result(result)
+        check_zulip_realm_query_result(result)
+        check_lear_realm_query_result(result)
+
+        with mock.patch(
+            "analytics.views.support.timezone_now",
+            return_value=timezone_now() - timedelta(minutes=50),
+        ):
+            self.client_post("/accounts/home/", {"email": self.nonreg_email("test")})
+            self.login("iago")
+            result = get_check_query_result(self.nonreg_email("test"), 1)
+            check_preregistration_user_query_result(result, self.nonreg_email("test"))
+            check_zulip_realm_query_result(result)
+
+            create_invitation("Denmark", self.nonreg_email("test1"))
+            result = get_check_query_result(self.nonreg_email("test1"), 1)
+            check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
+            check_zulip_realm_query_result(result)
+
+            email = self.nonreg_email("alice")
+            self.submit_realm_creation_form(
+                email, realm_subdomain="custom-test", realm_name="Zulip test"
+            )
+            result = get_check_query_result(email, 1)
+            check_realm_creation_query_result(result, email)
+
+            invite_expires_in_minutes = 10 * 24 * 60
+            do_create_multiuse_invite_link(
+                self.example_user("hamlet"),
+                invited_as=1,
+                invite_expires_in_minutes=invite_expires_in_minutes,
+            )
+            result = get_check_query_result("zulip", 2)
+            check_multiuse_invite_link_query_result(result)
+            check_zulip_realm_query_result(result)
+            MultiuseInvite.objects.all().delete()
+
+            do_send_realm_reactivation_email(get_realm("zulip"), acting_user=None)
+            result = get_check_query_result("zulip", 2)
+            check_realm_reactivation_link_query_result(result)
+            check_zulip_realm_query_result(result)
+
+            lear_nonreg_email = "newguy@lear.org"
+            self.client_post("/accounts/home/", {"email": lear_nonreg_email}, subdomain="lear")
+            result = get_check_query_result(lear_nonreg_email, 1)
+            check_preregistration_user_query_result(result, lear_nonreg_email)
+            check_lear_realm_query_result(result)
+
+            self.login_user(lear_user)
+            create_invitation("general", "newguy2@lear.org", lear_realm)
+            result = get_check_query_result("newguy2@lear.org", 1, lear_realm.string_id)
+            check_preregistration_user_query_result(result, "newguy2@lear.org", invite=True)
+            check_lear_realm_query_result(result)
+
+    def test_get_org_type_display_name(self) -> None:
+        self.assertEqual(get_org_type_display_name(Realm.ORG_TYPES["business"]["id"]), "Business")
+        self.assertEqual(get_org_type_display_name(883), "")
+
+    def test_unspecified_org_type_correctly_displayed(self) -> None:
+        """
+        Unspecified org type is special in that it is marked to not be shown
+        on the registration page (because organitions are not meant to be able to choose it),
+        but should be correctly shown at the /support/ endpoint.
+        """
+        realm = get_realm("zulip")
+
+        do_change_realm_org_type(realm, 0, acting_user=None)
+        self.assertEqual(realm.org_type, 0)
+
+        self.login("iago")
+
+        result = self.client_get("/activity/support", {"q": "zulip"}, subdomain="zulip")
+        self.assert_in_success_response(
+            [
+                f'<input type="hidden" name="realm_id" value="{realm.id}"',
+                '<option value="0" selected>',
+            ],
+            result,
+        )
+
+    @mock.patch("analytics.views.support.update_billing_method_of_current_plan")
+    def test_change_billing_method(self, m: mock.Mock) -> None:
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        result = self.client_post(
+            "/activity/support",
+            {"realm_id": f"{iago.realm_id}", "billing_method": "charge_automatically"},
+        )
+        m.assert_called_once_with(get_realm("zulip"), charge_automatically=True, acting_user=iago)
+        self.assert_in_success_response(
+            ["Billing method of zulip updated to charge automatically"], result
+        )
+
+        m.reset_mock()
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{iago.realm_id}", "billing_method": "send_invoice"}
+        )
+        m.assert_called_once_with(get_realm("zulip"), charge_automatically=False, acting_user=iago)
+        self.assert_in_success_response(
+            ["Billing method of zulip updated to pay by invoice"], result
+        )
+
+    def test_change_realm_plan_type(self) -> None:
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "2"}
+            )
+            m.assert_called_once_with(get_realm("zulip"), 2, acting_user=iago)
+            self.assert_in_success_response(
+                ["Plan type of zulip changed from self-hosted to limited"], result
+            )
+
+        with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{iago.realm_id}", "plan_type": "10"}
+            )
+            m.assert_called_once_with(get_realm("zulip"), 10, acting_user=iago)
+            self.assert_in_success_response(
+                ["Plan type of zulip changed from self-hosted to plus"], result
+            )
+
+    def test_change_org_type(self) -> None:
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "org_type": "70"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        with mock.patch("analytics.views.support.do_change_realm_org_type") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{iago.realm_id}", "org_type": "70"}
+            )
+            m.assert_called_once_with(get_realm("zulip"), 70, acting_user=iago)
+            self.assert_in_success_response(
+                ["Org type of zulip changed from Business to Government"], result
+            )
+
+    def test_attach_discount(self) -> None:
+        cordelia = self.example_user("cordelia")
+        lear_realm = get_realm("lear")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login("iago")
+
+        with mock.patch("analytics.views.support.attach_discount_to_realm") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
+            )
+            m.assert_called_once_with(get_realm("lear"), 25, acting_user=iago)
+            self.assert_in_success_response(["Discount of lear changed to 25% from 0%"], result)
+
+    def test_change_sponsorship_status(self) -> None:
+        lear_realm = get_realm("lear")
+        self.assertIsNone(get_customer_by_realm(lear_realm))
+
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "true"}
+        )
+        self.assert_in_success_response(["lear marked as pending sponsorship."], result)
+        customer = get_customer_by_realm(lear_realm)
+        assert customer is not None
+        self.assertTrue(customer.sponsorship_pending)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "sponsorship_pending": "false"}
+        )
+        self.assert_in_success_response(["lear is no longer pending sponsorship."], result)
+        customer = get_customer_by_realm(lear_realm)
+        assert customer is not None
+        self.assertFalse(customer.sponsorship_pending)
+
+    def test_approve_sponsorship(self) -> None:
+        lear_realm = get_realm("lear")
+        update_sponsorship_status(lear_realm, True, acting_user=None)
+        king_user = self.lear_user("king")
+        king_user.role = UserProfile.ROLE_REALM_OWNER
+        king_user.save()
+
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support",
+            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        result = self.client_post(
+            "/activity/support",
+            {"realm_id": f"{lear_realm.id}", "approve_sponsorship": "true"},
+        )
+        self.assert_in_success_response(["Sponsorship approved for lear"], result)
+        lear_realm.refresh_from_db()
+        self.assertEqual(lear_realm.plan_type, Realm.PLAN_TYPE_STANDARD_FREE)
+        customer = get_customer_by_realm(lear_realm)
+        assert customer is not None
+        self.assertFalse(customer.sponsorship_pending)
+        messages = UserMessage.objects.filter(user_profile=king_user)
+        self.assertIn(
+            "request for sponsored hosting has been approved", messages[0].message.content
+        )
+        self.assert_length(messages, 1)
+
+    def test_activate_or_deactivate_realm(self) -> None:
+        cordelia = self.example_user("cordelia")
+        lear_realm = get_realm("lear")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        self.login("iago")
+
+        with mock.patch("analytics.views.support.do_deactivate_realm") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "deactivated"}
+            )
+            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
+            self.assert_in_success_response(["lear deactivated"], result)
+
+        with mock.patch("analytics.views.support.do_send_realm_reactivation_email") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{lear_realm.id}", "status": "active"}
+            )
+            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
+            self.assert_in_success_response(
+                ["Realm reactivation email sent to admins of lear"], result
+            )
+
+    def test_change_subdomain(self) -> None:
+        cordelia = self.example_user("cordelia")
+        lear_realm = get_realm("lear")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new_name"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+        self.login("iago")
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/activity/support?q=new-name")
+        realm_id = lear_realm.id
+        lear_realm = get_realm("new-name")
+        self.assertEqual(lear_realm.id, realm_id)
+        self.assertTrue(Realm.objects.filter(string_id="lear").exists())
+        self.assertTrue(Realm.objects.filter(string_id="lear")[0].deactivated)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
+        )
+        self.assert_in_success_response(
+            ["Subdomain already in use. Please choose a different one."], result
+        )
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "zulip"}
+        )
+        self.assert_in_success_response(
+            ["Subdomain already in use. Please choose a different one."], result
+        )
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "lear"}
+        )
+        self.assert_in_success_response(
+            ["Subdomain already in use. Please choose a different one."], result
+        )
+
+        # Test renaming to a "reserved" subdomain
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "your-org"}
+        )
+        self.assert_in_success_response(
+            ["Subdomain reserved. Please choose a different one."], result
+        )
+
+    def test_downgrade_realm(self) -> None:
+        cordelia = self.example_user("cordelia")
+        self.login_user(cordelia)
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        iago = self.example_user("iago")
+        self.login_user(iago)
+
+        with mock.patch("analytics.views.support.downgrade_at_the_end_of_billing_cycle") as m:
+            result = self.client_post(
+                "/activity/support",
+                {
+                    "realm_id": f"{iago.realm_id}",
+                    "modify_plan": "downgrade_at_billing_cycle_end",
+                },
+            )
+            m.assert_called_once_with(get_realm("zulip"))
+            self.assert_in_success_response(
+                ["zulip marked for downgrade at the end of billing cycle"], result
+            )
+
+        with mock.patch(
+            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
+        ) as m:
+            result = self.client_post(
+                "/activity/support",
+                {
+                    "realm_id": f"{iago.realm_id}",
+                    "modify_plan": "downgrade_now_without_additional_licenses",
+                },
+            )
+            m.assert_called_once_with(get_realm("zulip"))
+            self.assert_in_success_response(
+                ["zulip downgraded without creating additional invoices"], result
+            )
+
+        with mock.patch(
+            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
+        ) as m1:
+            with mock.patch("analytics.views.support.void_all_open_invoices", return_value=1) as m2:
+                result = self.client_post(
+                    "/activity/support",
+                    {
+                        "realm_id": f"{iago.realm_id}",
+                        "modify_plan": "downgrade_now_void_open_invoices",
+                    },
+                )
+                m1.assert_called_once_with(get_realm("zulip"))
+                m2.assert_called_once_with(get_realm("zulip"))
+                self.assert_in_success_response(
+                    ["zulip downgraded and voided 1 open invoices"], result
+                )
+
+        with mock.patch("analytics.views.support.switch_realm_from_standard_to_plus_plan") as m:
+            result = self.client_post(
+                "/activity/support",
+                {
+                    "realm_id": f"{iago.realm_id}",
+                    "modify_plan": "upgrade_to_plus",
+                },
+            )
+            m.assert_called_once_with(get_realm("zulip"))
+            self.assert_in_success_response(["zulip upgraded to Plus"], result)
+
+    def test_scrub_realm(self) -> None:
+        cordelia = self.example_user("cordelia")
+        lear_realm = get_realm("lear")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        self.login("iago")
+
+        with mock.patch("analytics.views.support.do_scrub_realm") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{lear_realm.id}", "scrub_realm": "true"}
+            )
+            m.assert_called_once_with(lear_realm, acting_user=self.example_user("iago"))
+            self.assert_in_success_response(["lear scrubbed"], result)
+
+        with mock.patch("analytics.views.support.do_scrub_realm") as m:
+            result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
+            self.assert_json_error(result, "Invalid parameters")
+            m.assert_not_called()
+
+    def test_delete_user(self) -> None:
+        cordelia = self.example_user("cordelia")
+        hamlet = self.example_user("hamlet")
+        hamlet_email = hamlet.delivery_email
+        realm = get_realm("zulip")
+        self.login_user(cordelia)
+
+        result = self.client_post(
+            "/activity/support", {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id}
+        )
+        self.assertEqual(result.status_code, 302)
+        self.assertEqual(result["Location"], "/login/")
+
+        self.login("iago")
+
+        with mock.patch("analytics.views.support.do_delete_user_preserving_messages") as m:
+            result = self.client_post(
+                "/activity/support",
+                {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id},
+            )
+            m.assert_called_once_with(hamlet)
+            self.assert_in_success_response([f"{hamlet_email} in zulip deleted"], result)

analytics/urls.py

@@ -1,38 +1,51 @@
-from django.conf import settings
+from typing import List, Union
+
 from django.conf.urls import include
 from django.urls import path
 from django.urls.resolvers import URLPattern, URLResolver
 
+from analytics.views.installation_activity import (
+    get_installation_activity,
+    get_integrations_activity,
+)
+from analytics.views.realm_activity import get_realm_activity
+from analytics.views.remote_activity import get_remote_server_activity
 from analytics.views.stats import (
     get_chart_data,
     get_chart_data_for_installation,
     get_chart_data_for_realm,
+    get_chart_data_for_remote_installation,
+    get_chart_data_for_remote_realm,
     get_chart_data_for_stream,
     stats,
     stats_for_installation,
     stats_for_realm,
+    stats_for_remote_installation,
+    stats_for_remote_realm,
 )
+from analytics.views.support import remote_servers_support, support
+from analytics.views.user_activity import get_user_activity
 from zerver.lib.rest import rest_path
 
-i18n_urlpatterns: list[URLPattern | URLResolver] = [
+i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
     # Server admin (user_profile.is_staff) visible stats pages
+    path("activity", get_installation_activity),
+    path("activity/remote", get_remote_server_activity),
+    path("activity/integrations", get_integrations_activity),
+    path("activity/support", support, name="support"),
+    path("activity/remote/support", remote_servers_support, name="remote_servers_support"),
+    path("realm_activity/<realm_str>/", get_realm_activity),
+    path("user_activity/<user_profile_id>/", get_user_activity),
     path("stats/realm/<realm_str>/", stats_for_realm),
     path("stats/installation", stats_for_installation),
+    path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
+    path(
+        "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
+    ),
     # User-visible stats page
     path("stats", stats, name="stats"),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import stats_for_remote_installation, stats_for_remote_realm
-
-    i18n_urlpatterns += [
-        path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
-        path(
-            "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/",
-            stats_for_remote_realm,
-        ),
-    ]
-
 # These endpoints are a part of the API (V1), which uses:
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
@@ -47,25 +60,16 @@ v1_api_and_json_patterns = [
     rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
     rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
     rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/installation",
+        GET=get_chart_data_for_remote_installation,
+    ),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
+        GET=get_chart_data_for_remote_realm,
+    ),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import (
-        get_chart_data_for_remote_installation,
-        get_chart_data_for_remote_realm,
-    )
-
-    v1_api_and_json_patterns += [
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/installation",
-            GET=get_chart_data_for_remote_installation,
-        ),
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
-            GET=get_chart_data_for_remote_realm,
-        ),
-    ]
-
 i18n_urlpatterns += [
     path("api/v1/", include(v1_api_and_json_patterns)),
     path("json/", include(v1_api_and_json_patterns)),

analytics/views/activity_common.py

@@ -0,0 +1,198 @@
+import re
+import sys
+from datetime import datetime
+from typing import Any, Callable, Collection, Dict, List, Optional, Sequence, Union
+from urllib.parse import urlencode
+
+from django.conf import settings
+from django.db import connection
+from django.db.backends.utils import CursorWrapper
+from django.template import loader
+from django.urls import reverse
+from markupsafe import Markup
+from psycopg2.sql import Composable
+
+from zerver.lib.url_encoding import append_url_query_string
+from zerver.models import UserActivity, get_realm
+
+if sys.version_info < (3, 9):  # nocoverage
+    from backports import zoneinfo
+else:  # nocoverage
+    import zoneinfo
+
+eastern_tz = zoneinfo.ZoneInfo("America/New_York")
+
+
+if settings.BILLING_ENABLED:
+    pass
+
+
+def make_table(
+    title: str, cols: Sequence[str], rows: Sequence[Any], has_row_class: bool = False
+) -> str:
+    if not has_row_class:
+
+        def fix_row(row: Any) -> Dict[str, Any]:
+            return dict(cells=row, row_class=None)
+
+        rows = list(map(fix_row, rows))
+
+    data = dict(title=title, cols=cols, rows=rows)
+
+    content = loader.render_to_string(
+        "analytics/ad_hoc_query.html",
+        dict(data=data),
+    )
+
+    return content
+
+
+def get_page(
+    query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
+) -> Dict[str, str]:
+    cursor = connection.cursor()
+    cursor.execute(query)
+    rows = cursor.fetchall()
+    rows = list(map(list, rows))
+    cursor.close()
+
+    def fix_rows(
+        i: int, fixup_func: Union[Callable[[str], Markup], Callable[[datetime], str]]
+    ) -> None:
+        for row in rows:
+            row[i] = fixup_func(row[i])
+
+    total_row = []
+    for i, col in enumerate(cols):
+        if col == "Realm":
+            fix_rows(i, realm_activity_link)
+        elif col in ["Last time", "Last visit"]:
+            fix_rows(i, format_date_for_activity_reports)
+        elif col == "Hostname":
+            for row in rows:
+                row[i] = remote_installation_stats_link(row[0], row[i])
+        if len(totals_columns) > 0:
+            if i == 0:
+                total_row.append("Total")
+            elif i in totals_columns:
+                total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
+            else:
+                total_row.append("")
+    if len(totals_columns) > 0:
+        rows.insert(0, total_row)
+
+    content = make_table(title, cols, rows)
+
+    return dict(
+        content=content,
+        title=title,
+    )
+
+
+def dictfetchall(cursor: CursorWrapper) -> List[Dict[str, Any]]:
+    """Returns all rows from a cursor as a dict"""
+    desc = cursor.description
+    return [dict(zip((col[0] for col in desc), row)) for row in cursor.fetchall()]
+
+
+def format_date_for_activity_reports(date: Optional[datetime]) -> str:
+    if date:
+        return date.astimezone(eastern_tz).strftime("%Y-%m-%d %H:%M")
+    else:
+        return ""
+
+
+def user_activity_link(email: str, user_profile_id: int) -> Markup:
+    from analytics.views.user_activity import get_user_activity
+
+    url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
+    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
+
+
+def realm_activity_link(realm_str: str) -> Markup:
+    from analytics.views.realm_activity import get_realm_activity
+
+    url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
+    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
+
+
+def realm_stats_link(realm_str: str) -> Markup:
+    from analytics.views.stats import stats_for_realm
+
+    url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
+    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
+
+
+def realm_support_link(realm_str: str) -> Markup:
+    support_url = reverse("support")
+    query = urlencode({"q": realm_str})
+    url = append_url_query_string(support_url, query)
+    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
+
+
+def realm_url_link(realm_str: str) -> Markup:
+    url = get_realm(realm_str).uri
+    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
+
+
+def remote_installation_stats_link(server_id: int, hostname: str) -> Markup:
+    from analytics.views.stats import stats_for_remote_installation
+
+    url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
+    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a> {hostname}').format(
+        url=url, hostname=hostname
+    )
+
+
+def get_user_activity_summary(records: Collection[UserActivity]) -> Dict[str, Any]:
+    #: The type annotation used above is clearly overly permissive.
+    #: We should perhaps use TypedDict to clearly lay out the schema
+    #: for the user activity summary.
+    summary: Dict[str, Any] = {}
+
+    def update(action: str, record: UserActivity) -> None:
+        if action not in summary:
+            summary[action] = dict(
+                count=record.count,
+                last_visit=record.last_visit,
+            )
+        else:
+            summary[action]["count"] += record.count
+            summary[action]["last_visit"] = max(
+                summary[action]["last_visit"],
+                record.last_visit,
+            )
+
+    if records:
+        first_record = next(iter(records))
+        summary["name"] = first_record.user_profile.full_name
+        summary["user_profile_id"] = first_record.user_profile.id
+
+    for record in records:
+        client = record.client.name
+        query = str(record.query)
+
+        update("use", record)
+
+        if client == "API":
+            m = re.match("/api/.*/external/(.*)", query)
+            if m:
+                client = m.group(1)
+                update(client, record)
+
+        if client.startswith("desktop"):
+            update("desktop", record)
+        if client == "website":
+            update("website", record)
+        if ("send_message" in query) or re.search("/api/.*/external/.*", query):
+            update("send", record)
+        if query in [
+            "/json/update_pointer",
+            "/json/users/me/pointer",
+            "/api/v1/update_pointer",
+            "update_pointer_backend",
+        ]:
+            update("pointer", record)
+        update(client, record)
+
+    return summary

analytics/views/installation_activity.py

@@ -0,0 +1,420 @@
+import itertools
+import time
+from collections import defaultdict
+from contextlib import suppress
+from datetime import timedelta
+from typing import Dict, Optional, Tuple
+
+from django.conf import settings
+from django.db import connection
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from django.template import loader
+from django.utils.timezone import now as timezone_now
+from markupsafe import Markup
+from psycopg2.sql import SQL
+
+from analytics.lib.counts import COUNT_STATS
+from analytics.views.activity_common import (
+    dictfetchall,
+    get_page,
+    realm_activity_link,
+    realm_stats_link,
+    realm_support_link,
+    realm_url_link,
+)
+from analytics.views.support import get_plan_name
+from zerver.decorator import require_server_admin
+from zerver.lib.request import has_request_variables
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.models import Realm, UserActivityInterval, get_org_type_display_name
+
+if settings.BILLING_ENABLED:
+    from corporate.lib.stripe import (
+        estimate_annual_recurring_revenue_by_realm,
+        get_realms_to_default_discount_dict,
+    )
+
+
+def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
+    # To align with UTC days, we subtract an hour from end_time to
+    # get the start_time, since the hour that starts at midnight was
+    # on the previous day.
+    query = SQL(
+        """
+        select
+            r.string_id,
+            (now()::date - (end_time - interval '1 hour')::date) age,
+            coalesce(sum(value), 0) cnt
+        from zerver_realm r
+        join analytics_realmcount rc on r.id = rc.realm_id
+        where
+            property = 'messages_sent:is_bot:hour'
+        and
+            subgroup = 'false'
+        and
+            end_time > now()::date - interval '8 day' - interval '1 hour'
+        group by
+            r.string_id,
+            age
+    """
+    )
+    cursor = connection.cursor()
+    cursor.execute(query)
+    rows = dictfetchall(cursor)
+    cursor.close()
+
+    counts: Dict[str, Dict[int, int]] = defaultdict(dict)
+    for row in rows:
+        counts[row["string_id"]][row["age"]] = row["cnt"]
+
+    def format_count(cnt: int, style: Optional[str] = None) -> Markup:
+        if style is not None:
+            good_bad = style
+        elif cnt == min_cnt:
+            good_bad = "bad"
+        elif cnt == max_cnt:
+            good_bad = "good"
+        else:
+            good_bad = "neutral"
+
+        return Markup('<td class="number {good_bad}">{cnt}</td>').format(good_bad=good_bad, cnt=cnt)
+
+    result = {}
+    for string_id in counts:
+        raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
+        min_cnt = min(raw_cnts[1:])
+        max_cnt = max(raw_cnts[1:])
+
+        cnts = format_count(raw_cnts[0], "neutral") + Markup().join(map(format_count, raw_cnts[1:]))
+        result[string_id] = dict(cnts=cnts)
+
+    return result
+
+
+def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
+    now = timezone_now()
+
+    query = SQL(
+        """
+        SELECT
+            realm.string_id,
+            realm.date_created,
+            realm.plan_type,
+            realm.org_type,
+            coalesce(wau_table.value, 0) wau_count,
+            coalesce(dau_table.value, 0) dau_count,
+            coalesce(user_count_table.value, 0) user_profile_count,
+            coalesce(bot_count_table.value, 0) bot_count
+        FROM
+            zerver_realm as realm
+            LEFT OUTER JOIN (
+                SELECT
+                    value _14day_active_humans,
+                    realm_id
+                from
+                    analytics_realmcount
+                WHERE
+                    property = 'realm_active_humans::day'
+                    AND end_time = %(realm_active_humans_end_time)s
+            ) as _14day_active_humans_table ON realm.id = _14day_active_humans_table.realm_id
+            LEFT OUTER JOIN (
+                SELECT
+                    value,
+                    realm_id
+                from
+                    analytics_realmcount
+                WHERE
+                    property = '7day_actives::day'
+                    AND end_time = %(seven_day_actives_end_time)s
+            ) as wau_table ON realm.id = wau_table.realm_id
+            LEFT OUTER JOIN (
+                SELECT
+                    value,
+                    realm_id
+                from
+                    analytics_realmcount
+                WHERE
+                    property = '1day_actives::day'
+                    AND end_time = %(one_day_actives_end_time)s
+            ) as dau_table ON realm.id = dau_table.realm_id
+            LEFT OUTER JOIN (
+                SELECT
+                    value,
+                    realm_id
+                from
+                    analytics_realmcount
+                WHERE
+                    property = 'active_users_audit:is_bot:day'
+                    AND subgroup = 'false'
+                    AND end_time = %(active_users_audit_end_time)s
+            ) as user_count_table ON realm.id = user_count_table.realm_id
+            LEFT OUTER JOIN (
+                SELECT
+                    value,
+                    realm_id
+                from
+                    analytics_realmcount
+                WHERE
+                    property = 'active_users_audit:is_bot:day'
+                    AND subgroup = 'true'
+                    AND end_time = %(active_users_audit_end_time)s
+            ) as bot_count_table ON realm.id = bot_count_table.realm_id
+        WHERE
+            _14day_active_humans IS NOT NULL
+            or realm.plan_type = 3
+        ORDER BY
+            dau_count DESC,
+            string_id ASC
+    """
+    )
+
+    cursor = connection.cursor()
+    cursor.execute(
+        query,
+        {
+            "realm_active_humans_end_time": COUNT_STATS[
+                "realm_active_humans::day"
+            ].last_successful_fill(),
+            "seven_day_actives_end_time": COUNT_STATS["7day_actives::day"].last_successful_fill(),
+            "one_day_actives_end_time": COUNT_STATS["1day_actives::day"].last_successful_fill(),
+            "active_users_audit_end_time": COUNT_STATS[
+                "active_users_audit:is_bot:day"
+            ].last_successful_fill(),
+        },
+    )
+    rows = dictfetchall(cursor)
+    cursor.close()
+
+    for row in rows:
+        row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
+        row["age_days"] = int((now - row["date_created"]).total_seconds() / 86400)
+        row["is_new"] = row["age_days"] < 12 * 7
+
+    # get messages sent per day
+    counts = get_realm_day_counts()
+    for row in rows:
+        try:
+            row["history"] = counts[row["string_id"]]["cnts"]
+        except Exception:
+            row["history"] = ""
+
+    # estimate annual subscription revenue
+    total_arr = 0
+    if settings.BILLING_ENABLED:
+        estimated_arrs = estimate_annual_recurring_revenue_by_realm()
+        realms_to_default_discount = get_realms_to_default_discount_dict()
+
+        for row in rows:
+            row["plan_type_string"] = get_plan_name(row["plan_type"])
+
+            string_id = row["string_id"]
+
+            if string_id in estimated_arrs:
+                row["arr"] = estimated_arrs[string_id]
+
+            if row["plan_type"] in [Realm.PLAN_TYPE_STANDARD, Realm.PLAN_TYPE_PLUS]:
+                row["effective_rate"] = 100 - int(realms_to_default_discount.get(string_id, 0))
+            elif row["plan_type"] == Realm.PLAN_TYPE_STANDARD_FREE:
+                row["effective_rate"] = 0
+            elif (
+                row["plan_type"] == Realm.PLAN_TYPE_LIMITED
+                and string_id in realms_to_default_discount
+            ):
+                row["effective_rate"] = 100 - int(realms_to_default_discount[string_id])
+            else:
+                row["effective_rate"] = ""
+
+        total_arr += sum(estimated_arrs.values())
+
+    for row in rows:
+        row["org_type_string"] = get_org_type_display_name(row["org_type"])
+
+    # augment data with realm_minutes
+    total_hours = 0.0
+    for row in rows:
+        string_id = row["string_id"]
+        minutes = realm_minutes.get(string_id, 0.0)
+        hours = minutes / 60.0
+        total_hours += hours
+        row["hours"] = str(int(hours))
+        with suppress(Exception):
+            row["hours_per_user"] = "{:.1f}".format(hours / row["dau_count"])
+
+    # formatting
+    for row in rows:
+        row["realm_url"] = realm_url_link(row["string_id"])
+        row["stats_link"] = realm_stats_link(row["string_id"])
+        row["support_link"] = realm_support_link(row["string_id"])
+        row["string_id"] = realm_activity_link(row["string_id"])
+
+    # Count active sites
+    num_active_sites = sum(row["dau_count"] >= 5 for row in rows)
+
+    # create totals
+    total_dau_count = 0
+    total_user_profile_count = 0
+    total_bot_count = 0
+    total_wau_count = 0
+    for row in rows:
+        total_dau_count += int(row["dau_count"])
+        total_user_profile_count += int(row["user_profile_count"])
+        total_bot_count += int(row["bot_count"])
+        total_wau_count += int(row["wau_count"])
+
+    total_row = dict(
+        string_id="Total",
+        plan_type_string="",
+        org_type_string="",
+        effective_rate="",
+        arr=total_arr,
+        realm_url="",
+        stats_link="",
+        support_link="",
+        date_created_day="",
+        dau_count=total_dau_count,
+        user_profile_count=total_user_profile_count,
+        bot_count=total_bot_count,
+        hours=int(total_hours),
+        wau_count=total_wau_count,
+    )
+
+    rows.insert(0, total_row)
+
+    content = loader.render_to_string(
+        "analytics/realm_summary_table.html",
+        dict(
+            rows=rows,
+            num_active_sites=num_active_sites,
+            utctime=now.strftime("%Y-%m-%d %H:%M %Z"),
+            billing_enabled=settings.BILLING_ENABLED,
+        ),
+    )
+    return content
+
+
+def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
+    day_end = timestamp_to_datetime(time.time())
+    day_start = day_end - timedelta(hours=24)
+
+    output = Markup()
+    output += "Per-user online duration for the last 24 hours:\n"
+    total_duration = timedelta(0)
+
+    all_intervals = (
+        UserActivityInterval.objects.filter(
+            end__gte=day_start,
+            start__lte=day_end,
+        )
+        .select_related(
+            "user_profile",
+            "user_profile__realm",
+        )
+        .only(
+            "start",
+            "end",
+            "user_profile__delivery_email",
+            "user_profile__realm__string_id",
+        )
+        .order_by(
+            "user_profile__realm__string_id",
+            "user_profile__delivery_email",
+        )
+    )
+
+    by_string_id = lambda row: row.user_profile.realm.string_id
+    by_email = lambda row: row.user_profile.delivery_email
+
+    realm_minutes = {}
+
+    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
+        realm_duration = timedelta(0)
+        output += Markup("<hr>") + f"{string_id}\n"
+        for email, intervals in itertools.groupby(realm_intervals, by_email):
+            duration = timedelta(0)
+            for interval in intervals:
+                start = max(day_start, interval.start)
+                end = min(day_end, interval.end)
+                duration += end - start
+
+            total_duration += duration
+            realm_duration += duration
+            output += f"  {email:<37}{duration}\n"
+
+        realm_minutes[string_id] = realm_duration.total_seconds() / 60
+
+    output += f"\nTotal duration:                      {total_duration}\n"
+    output += f"\nTotal duration in minutes:           {total_duration.total_seconds() / 60.}\n"
+    output += f"Total duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}"
+    content = Markup("<pre>{}</pre>").format(output)
+    return content, realm_minutes
+
+
+@require_server_admin
+@has_request_variables
+def get_installation_activity(request: HttpRequest) -> HttpResponse:
+    duration_content, realm_minutes = user_activity_intervals()
+    counts_content: str = realm_summary_table(realm_minutes)
+    data = [
+        ("Counts", counts_content),
+        ("Durations", duration_content),
+    ]
+
+    title = "Activity"
+
+    return render(
+        request,
+        "analytics/activity.html",
+        context=dict(data=data, title=title, is_home=True),
+    )
+
+
+@require_server_admin
+def get_integrations_activity(request: HttpRequest) -> HttpResponse:
+    title = "Integrations by client"
+
+    query = SQL(
+        """
+        select
+            case
+                when query like '%%external%%' then split_part(query, '/', 5)
+                else client.name
+            end client_name,
+            realm.string_id,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            (query in ('send_message_backend', '/api/v1/send_message')
+            and client.name not in ('Android', 'ZulipiOS')
+            and client.name not like 'test: Zulip%%'
+            )
+        or
+            query like '%%external%%'
+        group by client_name, string_id
+        having max(last_visit) > now() - interval '2 week'
+        order by client_name, string_id
+    """
+    )
+
+    cols = [
+        "Client",
+        "Realm",
+        "Hits",
+        "Last time",
+    ]
+
+    integrations_activity = get_page(query, cols, title)
+
+    return render(
+        request,
+        "analytics/activity_details_template.html",
+        context=dict(
+            data=integrations_activity["content"],
+            title=integrations_activity["title"],
+            is_home=False,
+        ),
+    )

analytics/views/realm_activity.py

@@ -0,0 +1,245 @@
+import itertools
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+from django.db import connection
+from django.db.models import QuerySet
+from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
+from django.shortcuts import render
+from django.utils.timezone import now as timezone_now
+from psycopg2.sql import SQL
+
+from analytics.views.activity_common import (
+    format_date_for_activity_reports,
+    get_user_activity_summary,
+    make_table,
+    realm_stats_link,
+    user_activity_link,
+)
+from zerver.decorator import require_server_admin
+from zerver.models import Realm, UserActivity
+
+
+def get_user_activity_records_for_realm(realm: str, is_bot: bool) -> QuerySet[UserActivity]:
+    fields = [
+        "user_profile__full_name",
+        "user_profile__delivery_email",
+        "query",
+        "client__name",
+        "count",
+        "last_visit",
+    ]
+
+    records = UserActivity.objects.filter(
+        user_profile__realm__string_id=realm,
+        user_profile__is_active=True,
+        user_profile__is_bot=is_bot,
+    )
+    records = records.order_by("user_profile__delivery_email", "-last_visit")
+    records = records.select_related("user_profile", "client").only(*fields)
+    return records
+
+
+def realm_user_summary_table(
+    all_records: QuerySet[UserActivity], admin_emails: Set[str]
+) -> Tuple[Dict[str, Any], str]:
+    user_records = {}
+
+    def by_email(record: UserActivity) -> str:
+        return record.user_profile.delivery_email
+
+    for email, records in itertools.groupby(all_records, by_email):
+        user_records[email] = get_user_activity_summary(list(records))
+
+    def get_last_visit(user_summary: Dict[str, Dict[str, datetime]], k: str) -> Optional[datetime]:
+        if k in user_summary:
+            return user_summary[k]["last_visit"]
+        else:
+            return None
+
+    def get_count(user_summary: Dict[str, Dict[str, str]], k: str) -> str:
+        if k in user_summary:
+            return user_summary[k]["count"]
+        else:
+            return ""
+
+    def is_recent(val: datetime) -> bool:
+        age = timezone_now() - val
+        return age.total_seconds() < 5 * 60
+
+    rows = []
+    for email, user_summary in user_records.items():
+        email_link = user_activity_link(email, user_summary["user_profile_id"])
+        sent_count = get_count(user_summary, "send")
+        cells = [user_summary["name"], email_link, sent_count]
+        row_class = ""
+        for field in ["use", "send", "pointer", "desktop", "ZulipiOS", "Android"]:
+            visit = get_last_visit(user_summary, field)
+            if field == "use":
+                if visit and is_recent(visit):
+                    row_class += " recently_active"
+                if email in admin_emails:
+                    row_class += " admin"
+            val = format_date_for_activity_reports(visit)
+            cells.append(val)
+        row = dict(cells=cells, row_class=row_class)
+        rows.append(row)
+
+    def by_used_time(row: Dict[str, Any]) -> str:
+        return row["cells"][3]
+
+    rows = sorted(rows, key=by_used_time, reverse=True)
+
+    cols = [
+        "Name",
+        "Email",
+        "Total sent",
+        "Heard from",
+        "Message sent",
+        "Pointer motion",
+        "Desktop",
+        "ZulipiOS",
+        "Android",
+    ]
+
+    title = "Summary"
+
+    content = make_table(title, cols, rows, has_row_class=True)
+    return user_records, content
+
+
+def realm_client_table(user_summaries: Dict[str, Dict[str, Any]]) -> str:
+    exclude_keys = [
+        "internal",
+        "name",
+        "user_profile_id",
+        "use",
+        "send",
+        "pointer",
+        "website",
+        "desktop",
+    ]
+
+    rows = []
+    for email, user_summary in user_summaries.items():
+        email_link = user_activity_link(email, user_summary["user_profile_id"])
+        name = user_summary["name"]
+        for k, v in user_summary.items():
+            if k in exclude_keys:
+                continue
+            client = k
+            count = v["count"]
+            last_visit = v["last_visit"]
+            row = [
+                format_date_for_activity_reports(last_visit),
+                client,
+                name,
+                email_link,
+                count,
+            ]
+            rows.append(row)
+
+    rows = sorted(rows, key=lambda r: r[0], reverse=True)
+
+    cols = [
+        "Last visit",
+        "Client",
+        "Name",
+        "Email",
+        "Count",
+    ]
+
+    title = "Clients"
+
+    return make_table(title, cols, rows)
+
+
+def sent_messages_report(realm: str) -> str:
+    title = "Recently sent messages for " + realm
+
+    cols = [
+        "Date",
+        "Humans",
+        "Bots",
+    ]
+
+    # Uses index: zerver_message_realm_date_sent
+    query = SQL(
+        """
+        select
+            series.day::date,
+            user_messages.humans,
+            user_messages.bots
+        from (
+            select generate_series(
+                (now()::date - interval '2 week'),
+                now()::date,
+                interval '1 day'
+            ) as day
+        ) as series
+        left join (
+            select
+                date_sent::date date_sent,
+                count(*) filter (where not up.is_bot) as humans,
+                count(*) filter (where up.is_bot) as bots
+            from zerver_message m
+            join zerver_userprofile up on up.id = m.sender_id
+            join zerver_realm r on r.id = up.realm_id
+            where
+                r.string_id = %s
+            and
+                date_sent > now() - interval '2 week'
+            and
+                m.realm_id = r.id
+            group by
+                date_sent::date
+            order by
+                date_sent::date
+        ) user_messages on
+            series.day = user_messages.date_sent
+    """
+    )
+    cursor = connection.cursor()
+    cursor.execute(query, [realm])
+    rows = cursor.fetchall()
+    cursor.close()
+
+    return make_table(title, cols, rows)
+
+
+@require_server_admin
+def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
+    data: List[Tuple[str, str]] = []
+    all_user_records: Dict[str, Any] = {}
+
+    try:
+        admins = Realm.objects.get(string_id=realm_str).get_human_admin_users()
+    except Realm.DoesNotExist:
+        return HttpResponseNotFound()
+
+    admin_emails = {admin.delivery_email for admin in admins}
+
+    for is_bot, page_title in [(False, "Humans"), (True, "Bots")]:
+        all_records = get_user_activity_records_for_realm(realm_str, is_bot)
+
+        user_records, content = realm_user_summary_table(all_records, admin_emails)
+        all_user_records.update(user_records)
+
+        data += [(page_title, content)]
+
+    page_title = "Clients"
+    content = realm_client_table(all_user_records)
+    data += [(page_title, content)]
+
+    page_title = "History"
+    content = sent_messages_report(realm_str)
+    data += [(page_title, content)]
+
+    title = realm_str
+    realm_stats = realm_stats_link(realm_str)
+
+    return render(
+        request,
+        "analytics/activity.html",
+        context=dict(data=data, realm_stats_link=realm_stats, title=title),
+    )

analytics/views/remote_activity.py

@@ -0,0 +1,59 @@
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from psycopg2.sql import SQL
+
+from analytics.views.activity_common import get_page
+from zerver.decorator import require_server_admin
+
+
+@require_server_admin
+def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
+    title = "Remote servers"
+
+    query = SQL(
+        """
+        with icount as (
+            select
+                server_id,
+                max(value) as max_value,
+                max(end_time) as max_end_time
+            from zilencer_remoteinstallationcount
+            where
+                property='active_users:is_bot:day'
+                and subgroup='false'
+            group by server_id
+            ),
+        remote_push_devices as (
+            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
+            group by server_id
+        )
+        select
+            rserver.id,
+            rserver.hostname,
+            rserver.contact_email,
+            max_value,
+            push_user_count,
+            max_end_time
+        from zilencer_remotezulipserver rserver
+        left join icount on icount.server_id = rserver.id
+        left join remote_push_devices on remote_push_devices.server_id = rserver.id
+        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
+    """
+    )
+
+    cols = [
+        "ID",
+        "Hostname",
+        "Contact email",
+        "Analytics users",
+        "Mobile users",
+        "Last update time",
+    ]
+
+    remote_servers = get_page(query, cols, title, totals_columns=[3, 4])
+
+    return render(
+        request,
+        "analytics/activity_details_template.html",
+        context=dict(data=remote_servers["content"], title=remote_servers["title"], is_home=False),
+    )

analytics/views/stats.py

@@ -1,7 +1,7 @@
 import logging
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
-from typing import Any, Optional, TypeAlias, TypeVar, cast
+from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
 
 from django.conf import settings
 from django.db.models import QuerySet
@@ -10,6 +10,7 @@ from django.shortcuts import render
 from django.utils import translation
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
+from typing_extensions import TypeAlias
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
@@ -35,8 +36,7 @@ from zerver.lib.response import json_success
 from zerver.lib.streams import access_stream_by_id
 from zerver.lib.timestamp import convert_to_UTC
 from zerver.lib.validator import to_non_negative_int
-from zerver.models import Client, Realm, Stream, UserProfile
-from zerver.models.realms import get_realm
+from zerver.models import Client, Realm, Stream, UserProfile, get_realm
 
 if settings.ZILENCER_ENABLED:
     from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
@@ -51,9 +51,11 @@ def is_analytics_ready(realm: Realm) -> bool:
 def render_stats(
     request: HttpRequest,
     data_url_suffix: str,
-    realm: Realm | None,
+    realm: Optional[Realm],
     *,
-    title: str | None = None,
+    title: Optional[str] = None,
+    for_installation: bool = False,
+    remote: bool = False,
     analytics_ready: bool = True,
 ) -> HttpResponse:
     assert request.user.is_authenticated
@@ -73,20 +75,21 @@ def render_stats(
         guest_users = None
         space_used = None
 
+    page_params = dict(
+        data_url_suffix=data_url_suffix,
+        for_installation=for_installation,
+        remote=remote,
+        upload_space_used=space_used,
+        guest_users=guest_users,
+    )
+
     request_language = get_and_set_request_language(
         request,
         request.user.default_language,
         translation.get_language_from_path(request.path_info),
     )
 
-    # Sync this with stats_params_schema in base_page_params.ts.
-    page_params = dict(
-        page_type="stats",
-        data_url_suffix=data_url_suffix,
-        upload_space_used=space_used,
-        guest_users=guest_users,
-        translation_data=get_language_translation_data(request_language),
-    )
+    page_params["translation_data"] = get_language_translation_data(request_language)
 
     return render(
         request,
@@ -194,7 +197,7 @@ def get_chart_data_for_remote_realm(
 @require_server_admin
 def stats_for_installation(request: HttpRequest) -> HttpResponse:
     assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation")
+    return render_stats(request, "/installation", None, title="installation", for_installation=True)
 
 
 @require_server_admin
@@ -206,6 +209,8 @@ def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -
         f"/remote/{server.id}/installation",
         None,
         title=f"remote installation {server.hostname}",
+        for_installation=True,
+        remote=True,
     )
 
 
@@ -245,25 +250,25 @@ def get_chart_data(
     request: HttpRequest,
     user_profile: UserProfile,
     chart_name: str = REQ(),
-    min_length: int | None = REQ(converter=to_non_negative_int, default=None),
-    start: datetime | None = REQ(converter=to_utc_datetime, default=None),
-    end: datetime | None = REQ(converter=to_utc_datetime, default=None),
+    min_length: Optional[int] = REQ(converter=to_non_negative_int, default=None),
+    start: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
+    end: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
     # These last several parameters are only used by functions
     # wrapping get_chart_data; the callers are responsible for
     # parsing/validation/authorization for them.
-    realm: Realm | None = None,
+    realm: Optional[Realm] = None,
     for_installation: bool = False,
     remote: bool = False,
-    remote_realm_id: int | None = None,
+    remote_realm_id: Optional[int] = None,
     server: Optional["RemoteZulipServer"] = None,
-    stream: Stream | None = None,
+    stream: Optional[Stream] = None,
 ) -> HttpResponse:
-    TableType: TypeAlias = (
-        type["RemoteInstallationCount"]
-        | type[InstallationCount]
-        | type["RemoteRealmCount"]
-        | type[RealmCount]
-    )
+    TableType: TypeAlias = Union[
+        Type["RemoteInstallationCount"],
+        Type[InstallationCount],
+        Type["RemoteRealmCount"],
+        Type[RealmCount],
+    ]
     if for_installation:
         if remote:
             assert settings.ZILENCER_ENABLED
@@ -280,9 +285,9 @@ def get_chart_data(
         else:
             aggregate_table = RealmCount
 
-    tables: (
-        tuple[TableType] | tuple[TableType, type[UserCount]] | tuple[TableType, type[StreamCount]]
-    )
+    tables: Union[
+        Tuple[TableType], Tuple[TableType, Type[UserCount]], Tuple[TableType, Type[StreamCount]]
+    ]
 
     if chart_name == "number_of_humans":
         stats = [
@@ -291,7 +296,7 @@ def get_chart_data(
             COUNT_STATS["active_users_audit:is_bot:day"],
         ]
         tables = (aggregate_table,)
-        subgroup_to_label: dict[CountStat, dict[str | None, str]] = {
+        subgroup_to_label: Dict[CountStat, Dict[Optional[str], str]] = {
             stats[0]: {None: "_1day"},
             stats[1]: {None: "_15day"},
             stats[2]: {"false": "all_time"},
@@ -309,8 +314,8 @@ def get_chart_data(
         tables = (aggregate_table, UserCount)
         subgroup_to_label = {
             stats[0]: {
-                "public_stream": _("Public channels"),
-                "private_stream": _("Private channels"),
+                "public_stream": _("Public streams"),
+                "private_stream": _("Private streams"),
                 "private_message": _("Direct messages"),
                 "huddle_message": _("Group direct messages"),
             }
@@ -335,7 +340,7 @@ def get_chart_data(
     elif chart_name == "messages_sent_by_stream":
         if stream is None:
             raise JsonableError(
-                _("Missing channel for chart: {chart_name}").format(chart_name=chart_name)
+                _("Missing stream for chart: {chart_name}").format(chart_name=chart_name)
             )
         stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
         tables = (aggregate_table, StreamCount)
@@ -371,20 +376,18 @@ def get_chart_data(
         assert server is not None
         assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
         aggregate_table_remote = cast(
-            type[RemoteInstallationCount] | type[RemoteRealmCount], aggregate_table
+            Union[Type[RemoteInstallationCount], Type[RemoteRealmCount]], aggregate_table
         )  # https://stackoverflow.com/questions/68540528/mypy-assertions-on-the-types-of-types
         if not aggregate_table_remote.objects.filter(server=server).exists():
             raise JsonableError(
                 _("No analytics data available. Please contact your server administrator.")
             )
         if start is None:
-            first = (
-                aggregate_table_remote.objects.filter(server=server).order_by("remote_id").first()
-            )
+            first = aggregate_table_remote.objects.filter(server=server).first()
             assert first is not None
             start = first.end_time
         if end is None:
-            last = aggregate_table_remote.objects.filter(server=server).order_by("remote_id").last()
+            last = aggregate_table_remote.objects.filter(server=server).last()
             assert last is not None
             end = last.end_time
     else:
@@ -417,7 +420,7 @@ def get_chart_data(
 
     assert len({stat.frequency for stat in stats}) == 1
     end_times = time_range(start, end, stats[0].frequency, min_length)
-    data: dict[str, Any] = {
+    data: Dict[str, Any] = {
         "end_times": [int(end_time.timestamp()) for end_time in end_times],
         "frequency": stats[0].frequency,
     }
@@ -470,7 +473,7 @@ def get_chart_data(
     return json_success(request, data=data)
 
 
-def sort_by_totals(value_arrays: dict[str, list[int]]) -> list[str]:
+def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
     totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
     return [label for total, label in totals]
 
@@ -481,10 +484,12 @@ def sort_by_totals(value_arrays: dict[str, list[int]]) -> list[str]:
 # understanding the realm's traffic and the user's traffic. This function
 # tries to rank the clients so that taking the first N elements of the
 # sorted list has a reasonable chance of doing so.
-def sort_client_labels(data: dict[str, dict[str, list[int]]]) -> list[str]:
+def sort_client_labels(data: Dict[str, Dict[str, List[int]]]) -> List[str]:
     realm_order = sort_by_totals(data["everyone"])
     user_order = sort_by_totals(data["user"])
-    label_sort_values: dict[str, float] = {label: i for i, label in enumerate(realm_order)}
+    label_sort_values: Dict[str, float] = {}
+    for i, label in enumerate(realm_order):
+        label_sort_values[label] = i
     for i, label in enumerate(user_order):
         label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
     return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
@@ -493,7 +498,7 @@ def sort_client_labels(data: dict[str, dict[str, list[int]]]) -> list[str]:
 CountT = TypeVar("CountT", bound=BaseCount)
 
 
-def table_filtered_to_id(table: type[CountT], key_id: int) -> QuerySet[CountT]:
+def table_filtered_to_id(table: Type[CountT], key_id: int) -> QuerySet[CountT]:
     if table == RealmCount:
         return table._default_manager.filter(realm_id=key_id)
     elif table == UserCount:
@@ -524,9 +529,7 @@ def client_label_map(name: str) -> str:
     if name == "ZulipiOS":
         return "Old iOS app"
     if name == "ZulipMobile":
-        return "Mobile app (React Native)"
-    if name in ["ZulipFlutter", "ZulipMobile/flutter"]:
-        return "Mobile app beta (Flutter)"
+        return "Mobile app"
     if name in ["ZulipPython", "API: Python"]:
         return "Python API"
     if name.startswith("Zulip") and name.endswith("Webhook"):
@@ -534,8 +537,8 @@ def client_label_map(name: str) -> str:
     return name
 
 
-def rewrite_client_arrays(value_arrays: dict[str, list[int]]) -> dict[str, list[int]]:
-    mapped_arrays: dict[str, list[int]] = {}
+def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[int]]:
+    mapped_arrays: Dict[str, List[int]] = {}
     for label, array in value_arrays.items():
         mapped_label = client_label_map(label)
         if mapped_label in mapped_arrays:
@@ -548,18 +551,18 @@ def rewrite_client_arrays(value_arrays: dict[str, list[int]]) -> dict[str, list[
 
 def get_time_series_by_subgroup(
     stat: CountStat,
-    table: type[BaseCount],
+    table: Type[BaseCount],
     key_id: int,
-    end_times: list[datetime],
-    subgroup_to_label: dict[str | None, str],
+    end_times: List[datetime],
+    subgroup_to_label: Dict[Optional[str], str],
     include_empty_subgroups: bool,
-) -> dict[str, list[int]]:
+) -> Dict[str, List[int]]:
     queryset = (
         table_filtered_to_id(table, key_id)
         .filter(property=stat.property)
         .values_list("subgroup", "end_time", "value")
     )
-    value_dicts: dict[str | None, dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
+    value_dicts: Dict[Optional[str], Dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
     for subgroup, end_time, value in queryset:
         value_dicts[subgroup][end_time] = value
     value_arrays = {}

analytics/views/support.py

@@ -0,0 +1,451 @@
+import urllib
+from contextlib import suppress
+from dataclasses import dataclass
+from datetime import timedelta
+from decimal import Decimal
+from typing import Any, Dict, Iterable, List, Optional
+from urllib.parse import urlencode
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+from django.db.models import Q
+from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
+from django.shortcuts import render
+from django.urls import reverse
+from django.utils.timesince import timesince
+from django.utils.timezone import now as timezone_now
+from django.utils.translation import gettext as _
+
+from confirmation.models import Confirmation, confirmation_url
+from confirmation.settings import STATUS_USED
+from zerver.actions.create_realm import do_change_realm_subdomain
+from zerver.actions.realm_settings import (
+    do_change_realm_org_type,
+    do_change_realm_plan_type,
+    do_deactivate_realm,
+    do_scrub_realm,
+    do_send_realm_reactivation_email,
+)
+from zerver.actions.users import do_delete_user_preserving_messages
+from zerver.decorator import require_server_admin
+from zerver.forms import check_subdomain_available
+from zerver.lib.exceptions import JsonableError
+from zerver.lib.realm_icon import realm_icon_url
+from zerver.lib.request import REQ, has_request_variables
+from zerver.lib.subdomains import get_subdomain_from_hostname
+from zerver.lib.validator import check_bool, check_string_in, to_decimal, to_non_negative_int
+from zerver.models import (
+    MultiuseInvite,
+    PreregistrationRealm,
+    PreregistrationUser,
+    Realm,
+    RealmReactivationStatus,
+    UserProfile,
+    get_org_type_display_name,
+    get_realm,
+    get_user_profile_by_id,
+)
+from zerver.views.invite import get_invitee_emails_set
+
+if settings.ZILENCER_ENABLED:
+    from zilencer.models import RemoteZulipServer
+
+if settings.BILLING_ENABLED:
+    from corporate.lib.stripe import approve_sponsorship as do_approve_sponsorship
+    from corporate.lib.stripe import (
+        attach_discount_to_realm,
+        downgrade_at_the_end_of_billing_cycle,
+        downgrade_now_without_creating_additional_invoices,
+        get_discount_for_realm,
+        get_latest_seat_count,
+        make_end_of_cycle_updates_if_needed,
+        switch_realm_from_standard_to_plus_plan,
+        update_billing_method_of_current_plan,
+        update_sponsorship_status,
+        void_all_open_invoices,
+    )
+    from corporate.models import (
+        Customer,
+        CustomerPlan,
+        get_current_plan_by_realm,
+        get_customer_by_realm,
+    )
+
+
+def get_plan_name(plan_type: int) -> str:
+    return {
+        Realm.PLAN_TYPE_SELF_HOSTED: "self-hosted",
+        Realm.PLAN_TYPE_LIMITED: "limited",
+        Realm.PLAN_TYPE_STANDARD: "standard",
+        Realm.PLAN_TYPE_STANDARD_FREE: "open source",
+        Realm.PLAN_TYPE_PLUS: "plus",
+    }[plan_type]
+
+
+def get_confirmations(
+    types: List[int], object_ids: Iterable[int], hostname: Optional[str] = None
+) -> List[Dict[str, Any]]:
+    lowest_datetime = timezone_now() - timedelta(days=30)
+    confirmations = Confirmation.objects.filter(
+        type__in=types, object_id__in=object_ids, date_sent__gte=lowest_datetime
+    )
+    confirmation_dicts = []
+    for confirmation in confirmations:
+        realm = confirmation.realm
+        content_object = confirmation.content_object
+
+        type = confirmation.type
+        expiry_date = confirmation.expiry_date
+
+        assert content_object is not None
+        if hasattr(content_object, "status"):
+            if content_object.status == STATUS_USED:
+                link_status = "Link has been used"
+            else:
+                link_status = "Link has not been used"
+        else:
+            link_status = ""
+
+        now = timezone_now()
+        if expiry_date is None:
+            expires_in = "Never"
+        elif now < expiry_date:
+            expires_in = timesince(now, expiry_date)
+        else:
+            expires_in = "Expired"
+
+        url = confirmation_url(confirmation.confirmation_key, realm, type)
+        confirmation_dicts.append(
+            {
+                "object": confirmation.content_object,
+                "url": url,
+                "type": type,
+                "link_status": link_status,
+                "expires_in": expires_in,
+            }
+        )
+    return confirmation_dicts
+
+
+VALID_MODIFY_PLAN_METHODS = [
+    "downgrade_at_billing_cycle_end",
+    "downgrade_now_without_additional_licenses",
+    "downgrade_now_void_open_invoices",
+    "upgrade_to_plus",
+]
+
+VALID_STATUS_VALUES = [
+    "active",
+    "deactivated",
+]
+
+VALID_BILLING_METHODS = [
+    "send_invoice",
+    "charge_automatically",
+]
+
+
+@dataclass
+class PlanData:
+    customer: Optional["Customer"] = None
+    current_plan: Optional["CustomerPlan"] = None
+    licenses: Optional[int] = None
+    licenses_used: Optional[int] = None
+
+
+@require_server_admin
+@has_request_variables
+def support(
+    request: HttpRequest,
+    realm_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
+    plan_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
+    discount: Optional[Decimal] = REQ(default=None, converter=to_decimal),
+    new_subdomain: Optional[str] = REQ(default=None),
+    status: Optional[str] = REQ(default=None, str_validator=check_string_in(VALID_STATUS_VALUES)),
+    billing_method: Optional[str] = REQ(
+        default=None, str_validator=check_string_in(VALID_BILLING_METHODS)
+    ),
+    sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
+    approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
+    modify_plan: Optional[str] = REQ(
+        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
+    ),
+    scrub_realm: bool = REQ(default=False, json_validator=check_bool),
+    delete_user_by_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
+    query: Optional[str] = REQ("q", default=None),
+    org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
+) -> HttpResponse:
+    context: Dict[str, Any] = {}
+
+    if "success_message" in request.session:
+        context["success_message"] = request.session["success_message"]
+        del request.session["success_message"]
+
+    if settings.BILLING_ENABLED and request.method == "POST":
+        # We check that request.POST only has two keys in it: The
+        # realm_id and a field to change.
+        keys = set(request.POST.keys())
+        if "csrfmiddlewaretoken" in keys:
+            keys.remove("csrfmiddlewaretoken")
+        if len(keys) != 2:
+            raise JsonableError(_("Invalid parameters"))
+
+        assert realm_id is not None
+        realm = Realm.objects.get(id=realm_id)
+
+        acting_user = request.user
+        assert isinstance(acting_user, UserProfile)
+        if plan_type is not None:
+            current_plan_type = realm.plan_type
+            do_change_realm_plan_type(realm, plan_type, acting_user=acting_user)
+            msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(plan_type)} "
+            context["success_message"] = msg
+        elif org_type is not None:
+            current_realm_type = realm.org_type
+            do_change_realm_org_type(realm, org_type, acting_user=acting_user)
+            msg = f"Org type of {realm.string_id} changed from {get_org_type_display_name(current_realm_type)} to {get_org_type_display_name(org_type)} "
+            context["success_message"] = msg
+        elif discount is not None:
+            current_discount = get_discount_for_realm(realm) or 0
+            attach_discount_to_realm(realm, discount, acting_user=acting_user)
+            context[
+                "success_message"
+            ] = f"Discount of {realm.string_id} changed to {discount}% from {current_discount}%."
+        elif new_subdomain is not None:
+            old_subdomain = realm.string_id
+            try:
+                check_subdomain_available(new_subdomain)
+            except ValidationError as error:
+                context["error_message"] = error.message
+            else:
+                do_change_realm_subdomain(realm, new_subdomain, acting_user=acting_user)
+                request.session[
+                    "success_message"
+                ] = f"Subdomain changed from {old_subdomain} to {new_subdomain}"
+                return HttpResponseRedirect(
+                    reverse("support") + "?" + urlencode({"q": new_subdomain})
+                )
+        elif status is not None:
+            if status == "active":
+                do_send_realm_reactivation_email(realm, acting_user=acting_user)
+                context[
+                    "success_message"
+                ] = f"Realm reactivation email sent to admins of {realm.string_id}."
+            elif status == "deactivated":
+                do_deactivate_realm(realm, acting_user=acting_user)
+                context["success_message"] = f"{realm.string_id} deactivated."
+        elif billing_method is not None:
+            if billing_method == "send_invoice":
+                update_billing_method_of_current_plan(
+                    realm, charge_automatically=False, acting_user=acting_user
+                )
+                context[
+                    "success_message"
+                ] = f"Billing method of {realm.string_id} updated to pay by invoice."
+            elif billing_method == "charge_automatically":
+                update_billing_method_of_current_plan(
+                    realm, charge_automatically=True, acting_user=acting_user
+                )
+                context[
+                    "success_message"
+                ] = f"Billing method of {realm.string_id} updated to charge automatically."
+        elif sponsorship_pending is not None:
+            if sponsorship_pending:
+                update_sponsorship_status(realm, True, acting_user=acting_user)
+                context["success_message"] = f"{realm.string_id} marked as pending sponsorship."
+            else:
+                update_sponsorship_status(realm, False, acting_user=acting_user)
+                context["success_message"] = f"{realm.string_id} is no longer pending sponsorship."
+        elif approve_sponsorship:
+            do_approve_sponsorship(realm, acting_user=acting_user)
+            context["success_message"] = f"Sponsorship approved for {realm.string_id}"
+        elif modify_plan is not None:
+            if modify_plan == "downgrade_at_billing_cycle_end":
+                downgrade_at_the_end_of_billing_cycle(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
+            elif modify_plan == "downgrade_now_without_additional_licenses":
+                downgrade_now_without_creating_additional_invoices(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} downgraded without creating additional invoices"
+            elif modify_plan == "downgrade_now_void_open_invoices":
+                downgrade_now_without_creating_additional_invoices(realm)
+                voided_invoices_count = void_all_open_invoices(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
+            elif modify_plan == "upgrade_to_plus":
+                switch_realm_from_standard_to_plus_plan(realm)
+                context["success_message"] = f"{realm.string_id} upgraded to Plus"
+        elif scrub_realm:
+            do_scrub_realm(realm, acting_user=acting_user)
+            context["success_message"] = f"{realm.string_id} scrubbed."
+        elif delete_user_by_id:
+            user_profile_for_deletion = get_user_profile_by_id(delete_user_by_id)
+            user_email = user_profile_for_deletion.delivery_email
+            assert user_profile_for_deletion.realm == realm
+            do_delete_user_preserving_messages(user_profile_for_deletion)
+            context["success_message"] = f"{user_email} in {realm.subdomain} deleted."
+
+    if query:
+        key_words = get_invitee_emails_set(query)
+
+        case_insensitive_users_q = Q()
+        for key_word in key_words:
+            case_insensitive_users_q |= Q(delivery_email__iexact=key_word)
+        users = set(UserProfile.objects.filter(case_insensitive_users_q))
+        realms = set(Realm.objects.filter(string_id__in=key_words))
+
+        for key_word in key_words:
+            try:
+                URLValidator()(key_word)
+                parse_result = urllib.parse.urlparse(key_word)
+                hostname = parse_result.hostname
+                assert hostname is not None
+                if parse_result.port:
+                    hostname = f"{hostname}:{parse_result.port}"
+                subdomain = get_subdomain_from_hostname(hostname)
+                with suppress(Realm.DoesNotExist):
+                    realms.add(get_realm(subdomain))
+            except ValidationError:
+                users.update(UserProfile.objects.filter(full_name__iexact=key_word))
+
+        # full_names can have , in them
+        users.update(UserProfile.objects.filter(full_name__iexact=query))
+
+        context["users"] = users
+        context["realms"] = realms
+
+        confirmations: List[Dict[str, Any]] = []
+
+        preregistration_user_ids = [
+            user.id for user in PreregistrationUser.objects.filter(email__in=key_words)
+        ]
+        confirmations += get_confirmations(
+            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION],
+            preregistration_user_ids,
+            hostname=request.get_host(),
+        )
+
+        preregistration_realm_ids = [
+            user.id for user in PreregistrationRealm.objects.filter(email__in=key_words)
+        ]
+        confirmations += get_confirmations(
+            [Confirmation.REALM_CREATION],
+            preregistration_realm_ids,
+            hostname=request.get_host(),
+        )
+
+        multiuse_invite_ids = [
+            invite.id for invite in MultiuseInvite.objects.filter(realm__in=realms)
+        ]
+        confirmations += get_confirmations([Confirmation.MULTIUSE_INVITE], multiuse_invite_ids)
+
+        realm_reactivation_status_objects = RealmReactivationStatus.objects.filter(realm__in=realms)
+        confirmations += get_confirmations(
+            [Confirmation.REALM_REACTIVATION], [obj.id for obj in realm_reactivation_status_objects]
+        )
+
+        context["confirmations"] = confirmations
+
+        # We want a union of all realms that might appear in the search result,
+        # but not necessary as a separate result item.
+        # Therefore, we do not modify the realms object in the context.
+        all_realms = realms.union(
+            [
+                confirmation["object"].realm
+                for confirmation in confirmations
+                # For confirmations, we only display realm details when the type is USER_REGISTRATION
+                # or INVITATION.
+                if confirmation["type"] in (Confirmation.USER_REGISTRATION, Confirmation.INVITATION)
+            ]
+            + [user.realm for user in users]
+        )
+        plan_data: Dict[int, PlanData] = {}
+        for realm in all_realms:
+            current_plan = get_current_plan_by_realm(realm)
+            plan_data[realm.id] = PlanData(
+                customer=get_customer_by_realm(realm),
+                current_plan=current_plan,
+            )
+            if current_plan is not None:
+                new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(
+                    current_plan, timezone_now()
+                )
+                if last_ledger_entry is not None:
+                    if new_plan is not None:
+                        plan_data[realm.id].current_plan = new_plan
+                    else:
+                        plan_data[realm.id].current_plan = current_plan
+                    plan_data[realm.id].licenses = last_ledger_entry.licenses
+                    plan_data[realm.id].licenses_used = get_latest_seat_count(realm)
+        context["plan_data"] = plan_data
+
+    def get_realm_owner_emails_as_string(realm: Realm) -> str:
+        return ", ".join(
+            realm.get_human_owner_users()
+            .order_by("delivery_email")
+            .values_list("delivery_email", flat=True)
+        )
+
+    def get_realm_admin_emails_as_string(realm: Realm) -> str:
+        return ", ".join(
+            realm.get_human_admin_users(include_realm_owners=False)
+            .order_by("delivery_email")
+            .values_list("delivery_email", flat=True)
+        )
+
+    context["get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string
+    context["get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string
+    context["get_discount_for_realm"] = get_discount_for_realm
+    context["get_org_type_display_name"] = get_org_type_display_name
+    context["realm_icon_url"] = realm_icon_url
+    context["Confirmation"] = Confirmation
+    context["sorted_realm_types"] = sorted(
+        Realm.ORG_TYPES.values(), key=lambda d: d["display_order"]
+    )
+
+    return render(request, "analytics/support.html", context=context)
+
+
+def get_remote_servers_for_support(
+    email_to_search: Optional[str], hostname_to_search: Optional[str]
+) -> List["RemoteZulipServer"]:
+    if not email_to_search and not hostname_to_search:
+        return []
+
+    remote_servers_query = RemoteZulipServer.objects.order_by("id")
+    if email_to_search:
+        remote_servers_query = remote_servers_query.filter(contact_email__iexact=email_to_search)
+    elif hostname_to_search:
+        remote_servers_query = remote_servers_query.filter(hostname__icontains=hostname_to_search)
+
+    return list(remote_servers_query)
+
+
+@require_server_admin
+@has_request_variables
+def remote_servers_support(
+    request: HttpRequest, query: Optional[str] = REQ("q", default=None)
+) -> HttpResponse:
+    email_to_search = None
+    hostname_to_search = None
+    if query:
+        if "@" in query:
+            email_to_search = query
+        else:
+            hostname_to_search = query
+
+    remote_servers = get_remote_servers_for_support(
+        email_to_search=email_to_search, hostname_to_search=hostname_to_search
+    )
+    return render(
+        request,
+        "analytics/remote_server_support.html",
+        context=dict(
+            remote_servers=remote_servers,
+        ),
+    )

analytics/views/user_activity.py

@@ -0,0 +1,106 @@
+from typing import Any, Dict, List, Tuple
+
+from django.conf import settings
+from django.db.models import QuerySet
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+from analytics.views.activity_common import (
+    format_date_for_activity_reports,
+    get_user_activity_summary,
+    make_table,
+)
+from zerver.decorator import require_server_admin
+from zerver.models import UserActivity, UserProfile, get_user_profile_by_id
+
+if settings.BILLING_ENABLED:
+    pass
+
+
+def get_user_activity_records(
+    user_profile: UserProfile,
+) -> QuerySet[UserActivity]:
+    fields = [
+        "user_profile__full_name",
+        "query",
+        "client__name",
+        "count",
+        "last_visit",
+    ]
+
+    records = UserActivity.objects.filter(
+        user_profile=user_profile,
+    )
+    records = records.order_by("-last_visit")
+    records = records.select_related("user_profile", "client").only(*fields)
+    return records
+
+
+def raw_user_activity_table(records: QuerySet[UserActivity]) -> str:
+    cols = [
+        "query",
+        "client",
+        "count",
+        "last_visit",
+    ]
+
+    def row(record: UserActivity) -> List[Any]:
+        return [
+            record.query,
+            record.client.name,
+            record.count,
+            format_date_for_activity_reports(record.last_visit),
+        ]
+
+    rows = list(map(row, records))
+    title = "Raw data"
+    return make_table(title, cols, rows)
+
+
+def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
+    rows = []
+    for k, v in user_summary.items():
+        if k in ("name", "user_profile_id"):
+            continue
+        client = k
+        count = v["count"]
+        last_visit = v["last_visit"]
+        row = [
+            format_date_for_activity_reports(last_visit),
+            client,
+            count,
+        ]
+        rows.append(row)
+
+    rows = sorted(rows, key=lambda r: r[0], reverse=True)
+
+    cols = [
+        "last_visit",
+        "client",
+        "count",
+    ]
+
+    title = "User activity"
+    return make_table(title, cols, rows)
+
+
+@require_server_admin
+def get_user_activity(request: HttpRequest, user_profile_id: int) -> HttpResponse:
+    user_profile = get_user_profile_by_id(user_profile_id)
+    records = get_user_activity_records(user_profile)
+
+    data: List[Tuple[str, str]] = []
+    user_summary = get_user_activity_summary(records)
+    content = user_activity_summary_table(user_summary)
+
+    data += [("Summary", content)]
+
+    content = raw_user_activity_table(records)
+    data += [("Info", content)]
+
+    title = user_profile.delivery_email
+    return render(
+        request,
+        "analytics/activity.html",
+        context=dict(data=data, title=title),
+    )

api_docs/api-keys.md

@@ -1,89 +1,34 @@
 # API keys
 
-An **API key** is how a bot identifies itself to Zulip. For the official
-clients, such as the Python bindings, we recommend [downloading a `zuliprc`
-file](/api/configuring-python-bindings#download-a-zuliprc-file). This file
-contains an API key and other necessary configuration values for using the
-Zulip API with a specific account on a Zulip server.
+An **API key** is how a bot identifies itself to Zulip. Anyone with a
+bot's API key can impersonate the bot, so be careful with it!
 
 ## Get a bot's API key
 
-{start_tabs}
-
-{tab|desktop-web}
-
 {settings_tab|your-bots}
 
 1. Click **Active bots**.
 
 1. Find your bot. The bot's API key is under **API KEY**.
 
-{end_tabs}
-
-!!! warn ""
-
-    Anyone with a bot's API key can impersonate the bot, so be careful with it!
-
 ## Get your API key
 
-{start_tabs}
-
-{tab|desktop-web}
+Anyone with your API key can impersonate you, so be doubly careful with it.
 
 {settings_tab|account-and-privacy}
 
-1. Under **API key**, click **Manage your API key**.
+1. Under **API key**, click **Show/change your API key**.
 
 1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the instructions from there.
+   password, click **reset it** and follow the
+   instructions from there.
 
 1. Copy your API key.
 
-{end_tabs}
-
-!!! warn ""
-
-    Anyone with your API key can impersonate you, so be doubly careful with it.
-
-
 ## Invalidate an API key
 
-To invalidate an existing API key, you have to generate a new key.
+To invalidate a key, follow the instructions above, and click
+**Generate new API key** or click the **refresh**
+(<i class="fa fa-refresh"></i>) icon as appropriate.
 
-### Invalidate a bot's API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|your-bots}
-
-1. Click **Active bots**.
-
-1. Find your bot.
-
-1. Under **API KEY**, click the **refresh** (<i class="fa fa-refresh"></i>) icon
-   to the right of the bot's API key.
-
-{end_tabs}
-
-### Invalidate your API key
-
-{start_tabs}
-
-{tab|desktop-web}
-
-{settings_tab|account-and-privacy}
-
-1. Under **API key**, click **Manage your API key**.
-
-1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the instructions from there.
-
-1. Click **Generate new API key**
-
-{end_tabs}
-
-## Related articles
-
-* [Configuring the Python bindings](/api/configuring-python-bindings)
+This will generate a new key for you or the bot, and invalidate the old one.

api_docs/changelog.md

@@ -18,552 +18,8 @@ clients should check the `zulip_feature_level` field, present in the
 /register`](/api/register-queue) responses, to determine the API
 format used by the Zulip server that they are interacting with.
 
-## Changes in Zulip 9.0
-
-**Feature level 277**
-
-No changes; feature level used for Zulip 9.0 release.
-
-**Feature level 276**
-
-* [Markdown message formatting](/api/message-formatting#image-previews):
-  Image preview elements not contain a `data-original-dimensions`
-  attribute containing the dimensions of the original image.
-
-**Feature level 275**
-
-* [`POST /register`](/api/register-queue), [`PATCH
-  /settings`](/api/update-settings), [`PATCH
-  /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults):
-  Added new `web_animate_image_previews` setting, which controls how
-  animated images should be played in the web/desktop app message feed.
-
-**Feature level 274**
-
-* [`GET /events`](/api/get-events): `delete_message` events are now
-  always sent to the user who deletes the message, even if the message
-  was in a channel that the user was not subscribed to.
-
-**Feature level 273**
-
-* [`POST /register`](/api/register-queue): Added `server_thumbnail_formats`
-  describing what formats the server will thumbnail images into.
-
-**Feature level 272**
-
-* [`POST /user_uploads`](/api/upload-file): `uri` was renamed
-  to `url`, but remains available as a deprecated alias for
-  backwards-compatibility.
-
-**Feature level 271**
-
-* [`GET /messages`](/api/get-messages),
-  [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
-  [`POST /register`](/api/register-queue):
-  Added support for a new [search/narrow filter](/api/construct-narrow#changes)
-  operator, `with`, which uses a message ID for its operand. It returns
-  messages in the same conversation as the message with the specified
-  ID, and is designed to be used for creating permanent links to topics
-  that continue to work when a topic is moved or resolved.
-
-**Feature level 270**
-
-* `PATCH /realm`, [`POST /register`](/api/register-queue),
-  [`GET /events`](/api/get-events): Added two new realm settings,
-  `direct_message_initiator_group`, which is a
-  [group-setting value](/api/group-setting-values) describing the
-  set of users with permission to initiate direct message thread, and
-  `direct_message_permission_group`, which is a
-  [group-setting value](/api/group-setting-values) describing the
-  set of users of which at least one member must be included as sender
-  or recipient in all personal and group direct messages.
-  Removed `private_message_policy` property, as the permission to send
-  direct messages is now controlled by `direct_message_initiator_group`
-  and `direct_message_permission_group` settings.
-
-**Feature level 269**
-
-* [`POST /register`](/api/register-queue), [`PATCH
-  /settings`](/api/update-settings), [`PATCH
-  /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults):
-  Added new user setting `web_channel_default_view`, controlling the
-  behavior of clicking a channel link in the web/desktop apps.
-
-**Feature level 268**
-
-* [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults),
-  [`POST /register`](/api/register-queue), [`PATCH /settings`](/api/update-settings):
-  Added a new `web_navigate_to_sent_message` setting to allow users to decide
-  whether to automatically go to conversation where they sent a message.
-
-**Feature level 267**
-
-* [`GET /invites`](/api/get-invites),[`POST /invites`](/api/send-invites): Added
-  `notify_referrer_on_join` parameter, indicating whether the referrer has opted
-  to receive a direct message from the notification bot whenever a user joins
-  via this invitation.
-
-**Feature level 266**
-
-* `PATCH /realm`, [`POST /register`](/api/register-queue),
-  [`GET /events`](/api/get-events): Added `can_create_private_channel_group`
-  realm setting, which is a [group-setting value](/api/group-setting-values)
-  describing the set of users with permission to create private channels.
-* `PATCH /realm`, [`GET /events`](/api/get-events): Removed
-  `create_private_stream_policy` property, as the permission to create private
-  channels is now controlled by `can_create_private_channel_group` setting.
-* [`POST /register`](/api/register-queue): `realm_create_private_stream_policy`
-  field is deprecated, having been replaced by `can_create_private_channel_group`.
-  Notably, this backwards-compatible `realm_create_private_stream_policy` value
-  now contains the superset of the true value that best approximates the actual
-  permission setting.
-
-**Feature level 265**
-
-* [`GET /messages`](/api/get-messages),
-  [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
-  [`POST /register`](/api/register-queue):
-  Added a new [search/narrow filter](/api/construct-narrow#changes),
-  `is:followed`, matching messages in topics that the current user is
-  [following](/help/follow-a-topic).
-
-**Feature level 264**
-
-* `PATCH /realm`, [`POST /register`](/api/register-queue),
-  [`GET /events`](/api/get-events): Added `can_create_public_channel_group`
-  realm setting, which is a [group-setting value](/api/group-setting-values)
-  describing the set of users with permission to create channels.
-* `PATCH /realm`, [`GET /events`](/api/get-events): Removed
-  `create_public_stream_policy` property, as the permission to create public
-  channels is now controlled by `can_create_public_channel_group` setting.
-* [`POST /register`](/api/register-queue): `realm_create_public_stream_policy`
-  field is deprecated, having been replaced by `can_create_public_channel_group`.
-  Notably, this backwards-compatible `realm_create_public_stream_policy` value
-  now contains the superset of the true value that best approximates the actual
-  permission setting.
-
-**Feature level 263**
-
-* [`POST /users/me/presence`](/api/update-presence):
-  A new `last_update_id` parameter can be given, instructing
-  the server to only fetch presence data with `last_update_id`
-  greater than the value provided. The server also provides
-  a `presence_last_update_id` field in the response, which
-  tells the client the greatest `last_update_id` of the fetched
-  presence data. This can then be used as the value in the
-  aforementioned parameter to avoid re-fetching of already known
-  data when polling the endpoint next time.
-  Additionally, the client specifying the `last_update_id`
-  implies it uses the modern API format, so
-  `slim_presence=true` will be assumed by the server.
-
-
-* [`POST /register`](/api/register-queue): The response now also
-  includes a `presence_last_update_id` field, with the same
-  meaning as described above for [`/users/me/presence`](/api/update-presence).
-  In the same way, the retrieved value can be passed when
-  querying [`/users/me/presence`](/api/update-presence) to avoid
-  re-fetching of already known data.
-
-**Feature level 262**
-
-* [`GET /users/{user_id}/status`](/api/get-user-status): Added a new
-  endpoint to fetch an individual user's currently set
-  [status](/help/status-and-availability).
-
-**Feature level 261**
-
-* [`POST /invites`](/api/send-invites),
-  [`POST /invites/multiuse`](/api/create-invite-link): Added
-  `include_realm_default_subscriptions` parameter to indicate whether
-  the newly created user will be automatically subscribed to [default
-  channels](/help/set-default-channels-for-new-users) in the
-  organization. Previously, the default channel IDs needed to be included
-  in the `stream_ids` parameter. This also allows a newly created user
-  to be automatically subscribed to the default channels in an
-  organization when the user creating the invitation does not generally
-  have permission to [subscribe other users to
-  channels](/help/configure-who-can-invite-to-channels).
-
-**Feature level 260**
-
-* [`PATCH /user_groups/{user_group_id}`](/api/update-user-group):
-  Updating `can_mention_group` now uses a race-resistant format where
-  the client sends the expected `old` value and desired `new` value.
-
-**Feature level 259**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events):
-  For the `onboarding_steps` event type, an array of onboarding steps
-  to be displayed to clients is sent. Onboarding step now has one-time
-  notices as the only valid type. Prior to this, both hotspots and
-  one-time notices were valid types of onboarding steps. There is no compatibility
-  support, as we expect that only official Zulip clients will interact with
-  this data. Currently, no client other than the Zulip web app uses this.
-
-**Feature level 258**
-
-* [`GET /user_groups`](/api/get-user-groups), [`POST
-  /register`](/api/register-queue): `can_mention_group` field can now
-  either be an ID of a named user group with the permission, or an
-  object describing the set of users and groups with the permission.
-* [`POST /user_groups/create`](/api/create-user-group), [`PATCH
-  /user_groups/{user_group_id}`](/api/update-user-group): The
-  `can_mention_group` parameter can now either be an ID of a named
-  user group or an object describing a set of users and groups.
-
-**Feature level 257**
-
-* [`POST /register`](/api/register-queue),
-  [`POST /server_settings`](/api/get-server-settings), `PATCH /realm`:
-  `realm_uri` was renamed to `realm_url`, but remains available as a
-  deprecated alias for backwards-compatibility.
-* Mobile push notification payloads, similarly, have a new `realm_url`,
-  replacing `realm_uri`, which remains available as a deprecated alias
-  for backwards-compatibility.
-
-**Feature level 256**
-
-* [`POST /streams/{stream_id}/delete_topic`](/api/delete-topic),
-  [`GET /events`](/api/get-events): When messages are deleted, a
-  [`stream` op: `update`](/api/get-events#stream-update) event with
-  an updated value for `first_message_id` may now be sent to clients.
-
-**Feature level 255**
-
-* "Stream" was renamed to "Channel" across strings in the Zulip API
-  and UI. Clients supporting a range of server versions are encouraged
-  to use different strings based on the server's API feature level for
-  consistency. Note that feature level marks the strings transition
-  only: Actual API changes related to this transition have their own
-  API changelog entries.
-
-**Feature level 254**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events),
-  [`GET /streams`](/api/get-streams),
-  [`GET /streams/{stream_id}`](/api/get-stream-by-id),
-  [`GET /users/me/subscriptions`](/api/get-subscriptions): Added a new
-  field `creator_id` to stream and subscription objects, which contains the
-  user ID of the stream's creator.
-
-**Feature level 253**
-
-* [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults),
-  [`POST /register`](/api/register-queue), [`PATCH /settings`](/api/update-settings):
-  Added new `receives_typing_notifications` option to allow users to decide whether
-  to receive typing notification events from other users.
-
-**Feature level 252**
-
-* `PATCH /realm/profile_fields/{field_id}`: `name`, `hint`, `display_in_profile_summary`,
-  `required` and `field_data` fields are now optional during an update. Previously we
-  required the clients to populate the fields in the PATCH request even if there was
-  no change to those fields' values.
-
-**Feature level 251**
-
-* [`POST /register`](/api/register-queue): Fixed `realm_upload_quota_mib`
-  value to actually be in MiB. Until now the value was in bytes.
-
-**Feature level 250**
-
-* [`GET /messages`](/api/get-messages),
-  [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
-  [`POST /register`](/api/register-queue):
-  Added support for two [search/narrow filters](/api/construct-narrow#changes)
-  related to stream messages: `channel` and `channels`. The `channel`
-  operator is an alias for the `stream` operator. The `channels`
-  operator is an alias for the `streams` operator.
-
-**Feature level 249**
-
-* [`GET /messages`](/api/get-messages),
-  [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
-  [`POST /register`](/api/register-queue):
-  Added support for a new [search/narrow filter](/api/construct-narrow#changes),
-  `has:reaction`, which returns messages with at least one [emoji
-  reaction](/help/emoji-reactions).
-
-**Feature level 248**
-
-* [`POST /typing`](/api/set-typing-status), [`POST /messages`](/api/send-message),
-  [`POST /scheduled_messages`](/api/create-scheduled-message),
-  [`PATCH /scheduled_messages/<int:scheduled_message_id>`](/api/update-scheduled-message):
-  Added `"channel"` as an additional value for the `type` parameter to
-  indicate a stream message.
-
-**Feature level 247**
-
-* [Markdown message formatting](/api/message-formatting#mentions):
-  Added `channel` to the supported options for [wildcard
-  mentions](/help/mention-a-user-or-group#mention-everyone-on-a-stream).
-
-**Feature level 246**
-
-* [`POST /register`](/api/register-queue), [`POST
-  /events`](/api/get-events): Added new `require_unique_names` setting
-  controlling whether users names can duplicate others.
-
-**Feature level 245**
-
-* [`PATCH
-  /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults)
-  [`POST /register`](/api/register-queue), [`GET
-  /events`](/api/get-events), [`PATCH
-  /settings`](/api/update-settings): Added new `web_font_size_px` and
-  `web_line_height_percent` settings to allow users to control the
-  styling of the web application.
-
-**Feature level 244**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events),
-  [`POST /realm/profile_fields`](/api/create-custom-profile-field),
-  [`GET /realm/profile_fields`](/api/get-custom-profile-fields): Added a new
-  parameter `required`, on custom profile field objects, indicating whether an
-  organization administrator has configured the field as something users should
-  be required to provide.
-
-**Feature level 243**
-
-* [`POST /register`](/api/register-queue), [`GET
-  /events`](/api/get-events): Changed the format of
-  `realm_authentication_methods` and `authentication_methods`,
-  respectively, to use a dictionary rather than a boolean as the value
-  for each authentication method. The new dictionaries are more
-  extensively and contain fields indicating whether the backend is
-  unavailable to the current realm due to Zulip Cloud plan
-  restrictions or any other reason.
-
-**Feature level 242**
-
-* [`POST /register`](/api/register-queue), [`POST /events`](/api/get-events),
-  `PATCH /realm`: Added `zulip_update_announcements_stream_id` realm setting,
-  which is the ID of the of the stream to which automated messages announcing
-  new features or other end-user updates about the Zulip software are sent.
-
-**Feature level 241**
-
-* [`POST /register`](/api/register-queue), [`POST /events`](/api/get-events),
-  `PATCH /realm`: Renamed the realm settings `notifications_stream` and
-  `signup_notifications_stream` to `new_stream_announcements_stream` and
-  `signup_announcements_stream`, respectively.
-
-**Feature level 240**
-
-* [`GET /events`](/api/get-events): The `restart` event no longer contains an
-  optional `immediate` flag.
-* [`GET /events`](/api/get-events): A new `web_reload_client` event has been
-  added; it is used to signal to website-based clients that they should reload
-  their code.  This was previously implied by the `restart` event.
-
-Feature levels 238-239 are reserved for future use in 8.x maintenance
-releases.
-
 ## Changes in Zulip 8.0
 
-**Feature level 237**
-
-No changes; feature level used for Zulip 8.0 release.
-
-**Feature level 236**
-
-* [`POST /messages`](/api/send-message), [`POST
-  /scheduled_messages`](/api/create-scheduled-message): The new
-  `read_by_sender` parameter lets the client override the heuristic
-  that determines whether the new message will be initially marked
-  read by its sender.
-
-**Feature level 235**
-
-* [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults),
-  [`POST /register`](/api/register-queue), [`PATCH /settings`](/api/update-settings):
-  Added a new user setting, `automatically_follow_topics_where_mentioned`,
-  that allows the user to automatically follow topics where the user is mentioned.
-
-**Feature level 234**
-
-* Mobile push notifications now include a `realm_name` field.
-* [`POST /mobile_push/test_notification`](/api/test-notify) now sends
-  a test notification with `test` rather than `test-by-device-token`
-  in the `event` field.
-
-**Feature level 233**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events):
-  Renamed the `hotspots` event type and the related `hotspots` object array
-  to `onboarding_steps`. These are sent to clients if there are onboarding
-  steps to display to the user. Onboarding steps now include
-  both hotspots and one-time notices. Prior to this, hotspots were the only
-  type of onboarding step. Also, added a `type` field to the objects
-  returned in the renamed `onboarding_steps` array to distinguish between
-  the two types of onboarding steps.
-
-* `POST /users/me/onboarding_steps`: Added a new endpoint, which
-  deprecates the `/users/me/hotspots` endpoint, in order to support
-  displaying both one-time notices (which highlight new features for
-  existing users) and hotspots (which are used in new user tutorials).
-  This endpoint marks both types of onboarding steps, i.e. `hotspot`
-  and `one_time_notice`, as read by the user. There is no compatibility
-  support for `/users/me/hotspots` as no client other than the Zulip
-  web app used the endpoint prior to these changes.
-
-**Feature level 232**
-
-* [`POST /register`](/api/register-queue): Added a new
-  `user_list_incomplete` [client
-  capability](/api/register-queue#parameter-client_capabilities)
-  controlling whether `realm_users` contains "Unknown user"
-  placeholder objects for users that the current user cannot access
-  due to a `can_access_all_users_group` policy.
-
-* [`GET /events`](/api/get-events): The new `user_list_incomplete`
-  [client
-  capability](/api/register-queue#parameter-client_capabilities)
-  controls whether to send `realm_user` events with `op: "add"`
-  containing "Unknown user" placeholder objects to clients when a new
-  user is created that the client does not have access to due to a
-  `can_access_all_users_group` policy.
-
-**Feature level 231**
-
-* [`POST /register`](/api/register-queue):
-  `realm_push_notifications_enabled` now represents more accurately
-  whether push notifications are actually enabled via the mobile push
-  notifications service. Added
-  `realm_push_notifications_enabled_end_timestamp` field to realm
-  data.
-
-* [`GET /events`](/api/get-events): A `realm` update event is now sent
-  whenever `push_notifications_enabled` or
-  `push_notifications_enabled_end_timestamp` changes.
-
-**Feature level 230**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events):
-  Added `has_trigger` field to objects returned in the `hotspots` array to
-  identify if the hotspot will activate only when some specific event
-  occurs.
-
-**Feature level 229**
-
-* [`PATCH /messages/{message_id}`](/api/update-message), [`POST
-  /messages`](/api/send-message): Topic wildcard mentions involving
-  large numbers of participants are now restricted by
-  `wildcard_mention_policy`. The server now uses the
-  `STREAM_WILDCARD_MENTION_NOT_ALLOWED` and
-  `TOPIC_WILDCARD_MENTION_NOT_ALLOWED` error codes when a message is
-  rejected because of `wildcard_mention_policy`.
-
-**Feature level 228**
-
-* [`GET /events`](/api/get-events): `realm_user` events with `op: "update"`
-  are now only sent to users who can access the modified user.
-
-* [`GET /events`](/api/get-events): `presence` events are now only sent to
-  users who can access the user who comes back online if the
-  `CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE` server setting is set
-  to `true`.
-
-* [`GET /events`](/api/get-events): `user_status` events are now only
-  sent to users who can access the modified user.
-
-* [`GET /realm/presence`](/api/get-presence): The endpoint now returns
-  presence information of accessible users only if the
-  `CAN_ACCESS_ALL_USERS_GROUP_LIMITS_PRESENCE` server setting is set
-  to `true`.
-
-* [`GET /events`](/api/get-events): `realm_user` events with `op: "add"`
-  are now also sent when a guest user gains access to a user.
-
-* [`GET /events`](/api/get-events): `realm_user` events with `op: "remove"`
-  are now also sent when a guest user loses access to a user.
-
-**Feature level 227**
-
-* [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults),
-  [`POST /register`](/api/register-queue), [`PATCH /settings`](/api/update-settings):
-  Added `DMs, mentions, and followed topics` option for `desktop_icon_count_display`
-  setting, and renumbered the options.
-  The total unread count of DMs, mentions, and followed topics appears in
-  desktop sidebar and browser tab when this option is configured.
-
-**Feature level 226**
-
-* [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events),
-  [`GET /users/me/subscriptions`](/api/get-subscriptions): Removed
-  `email_address` field from subscription objects.
-
-* [`GET /streams/{stream_id}/email_address`](/api/get-stream-email-address):
-  Added new endpoint to get email address of a stream.
-
-**Feature level 225**
-
-* `PATCH /realm`, [`POST /register`](/api/register-queue),
-  [`GET /events`](/api/get-events): Added `can_access_all_users_group_id`
-  realm setting, which is the ID of the user group whose members can
-  access all the users in the oragnization.
-
-* [`POST /register`](/api/register-queue): Added `allowed_system_groups`
-  field to configuration data object of permission settings passed in
-  `server_supported_permission_settings`.
-
-**Feature level 224**
-
-* [`GET /events`](/api/get-events), [`GET /messages`](/api/get-messages),
-  [`GET /messages/{message_id}`](/api/get-message): Of the [available
-  message flags](/api/update-message-flags#available-flags) that a user
-  may have for a message, the `wildcard_mentioned` flag was
-  deprecated in favor of the `stream_wildcard_mentioned` and
-  `topic_wildcard_mentioned` flags, but it is still available for
-  backwards compatibility.
-
-**Feature level 223**
-
-* [`POST /users/me/apns_device_token`](/api/add-apns-token):
-  The `appid` parameter is now required.
-  Previously it defaulted to the server setting `ZULIP_IOS_APP_ID`,
-  defaulting to "org.zulip.Zulip".
-
-* `POST /remotes/server/register`: The `ios_app_id` parameter is now
-  required when `kind` is 1, i.e. when registering an APNs token.
-  Previously it was ignored, and the push bouncer effectively
-  assumed its value was the server setting `APNS_TOPIC`,
-  defaulting to "org.zulip.Zulip".
-
-**Feature level 222**
-
-* [`GET /events`](/api/get-events): When a user is deactivated or
-  reactivated, the server uses `realm_user` events with `op: "update"`
-  updating the `is_active` field, instead of `realm_user` events with
-  `op: "remove"` and `op: "add"`, respectively.
-
-* [`GET /events`](/api/get-events): When a bot is deactivated or
-  reactivated, the server sends `realm_bot` events with `op: "update"`
-  updating the `is_active` field, instead of `realm_bot` events with
-  `op: "remove"` and `op: "add"`, respectively.
-
-**Feature level 221**
-
-* [`POST /register`](/api/register-queue): Added `server_supported_permission_settings`
-  field in the response which contains configuration data for various permission
-  settings.
-
-**Feature level 220**
-
-* [`GET /events`](/api/get-events): Stream creation events for web-public
-  streams are now sent to all guest users in the organization as well.
-
-* [`GET /events`](/api/get-events): The `subscription` events for `op:
-  "peer_add"` and `op: "peer_remove"` are now sent to subscribed guest
-  users for public streams and to all the guest users for web-public
-  streams; previously, they incorrectly only received these for
-  private streams.
-
 **Feature level 219**
 
 * [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults)
@@ -571,10 +27,6 @@ No changes; feature level used for Zulip 8.0 release.
   [`PATCH /settings`](/api/update-settings): Renamed `default_view` and
   `escape_navigates_to_default_view` settings to `web_home_view` and
   `web_escape_navigates_to_home_view` respectively.
-* [`POST /user_topics`](/api/update-user-topic), [`POST
-  register`](/api/register-queue), [`GET /events`](/api/get-events):
-  Added followed as a supported value for visibility policies in
-  `user_topic` objects.
 
 **Feature level 218**
 
@@ -588,7 +40,7 @@ No changes; feature level used for Zulip 8.0 release.
 * [`POST /mobile_push/test_notification`](/api/test-notify): Added new endpoint
   to send a test push notification to a mobile device or devices.
 
-**Feature level 216**
+**Feature level 216**:
 
 * `PATCH /realm`, [`POST register`](/api/register-queue),
   [`GET /events`](/api/get-events): Added `enable_guest_user_indicator`
@@ -666,19 +118,18 @@ No changes; feature level used for Zulip 8.0 release.
   to an organization. Previously, only admin users could create these
   links.
 
-* [`POST /invites/multiuse`](/api/create-invite-link): Non-admin users can
-  now use this endpoint to create reusable invitation links. Previously,
-  this endpoint was restricted to admin users only.
+* `POST /invites/multiuse`: Non-admin users can now use this endpoint
+  to create reusable invitation links. Previously, this endpoint was
+  restricted to admin users only.
 
-* [`GET /invites`](/api/get-invites): Endpoint response for non-admin users now
-  includes both email invitations and reusable invitation links that they have
-  created. Previously, non-admin users could only create email invitations, and
-  therefore the response did not include reusable invitation links for these
-  users.
+* `GET /invites`: Endpoint response for non-admin users now includes both
+  email invitations and reusable invitation links that they have created.
+  Previously, non-admin users could only create email invitations, and
+  therefore the response did not include reusable invitation links for these users.
 
-* [`DELETE /invites/multiuse/{invite_id}`](/api/revoke-invite-link): Non-admin
-  users can now revoke reusable invitation links they have created. Previously,
-  only admin users could create and revoke reusable invitation links.
+* `DELETE /invites/multiuse/{invite_id}`: Non-admin users can now revoke
+  reusable invitation links they have created. Previously, only admin users could
+  create and revoke reusable invitation links.
 
 * [`GET /events`](/api/get-events): When the set of invitations in an
   organization changes, an `invites_changed` event is now sent to the
@@ -799,10 +250,10 @@ No changes; feature level used for Zulip 8.0 release.
 
 * [`GET /messages`](/api/get-messages),
   [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
+  [`POST /message/flags/narrow`](/api/update-message-flags-for-narrow),
   [`POST /register`](/api/register-queue):
-  For [search/narrow filters](/api/construct-narrow#message-ids) with the
-  `id` operator, added support for encoding the message ID operand as either
+  For [search/narrow filters](/api/construct-narrow) with the `id`
+  operator, added support for encoding the message ID operand as either
   a string or an integer. Previously, only string encoding was supported.
 
 **Feature level 193**
@@ -904,8 +355,8 @@ No changes; feature level used for Zulip 7.0 release.
 
 **Feature level 180**
 
-* [`POST /invites`](/api/send-invites): Added support for invitations specifying
-  the empty list as the user's initial stream subscriptions. Previously, this
+* `POST /invites`: Added support for invitations specifying the empty
+  list as the user's initial stream subscriptions. Previously, this
   returned an error. This change was also backported to Zulip 6.2, and
   is available at feature levels 157-158 as well.
 
@@ -922,7 +373,7 @@ No changes; feature level used for Zulip 7.0 release.
 
 **Feature level 178**
 
-* [`POST /users/me/presence`](/api/update-presence),
+* `POST /users/me/presence`,
   [`GET /users/<user_id_or_email>/presence`](/api/get-user-presence),
   [`GET /realm/presence`](/api/get-presence),
   [`POST /register`](/api/register-queue),
@@ -935,9 +386,9 @@ No changes; feature level used for Zulip 7.0 release.
 
 * [`GET /messages`](/api/get-messages),
   [`GET /messages/matches_narrow`](/api/check-messages-match-narrow),
-  [`POST /messages/flags/narrow`](/api/update-message-flags-for-narrow),
+  [`POST /message/flags/narrow`](/api/update-message-flags-for-narrow),
   [`POST /register`](/api/register-queue):
-  Added support for three [search/narrow filters](/api/construct-narrow#changes)
+  Added support for three [search/narrow filters](/api/construct-narrow)
   related to direct messages: `is:dm`, `dm` and `dm-including`.
   The `dm` operator replaces and deprecates the `pm-with` operator.
   The `is:dm` filter replaces and deprecates the `is:private` filter.
@@ -972,7 +423,7 @@ No changes; feature level used for Zulip 7.0 release.
   determine the user's preference on whether to mark messages as read or not when
   scrolling through their message feed.
 
-**Feature level 174**
+**Feature level 174**:
 
 * [`POST /typing`](/api/set-typing-status), [`POST /messages`](/api/send-message):
   Added `"direct"` as the preferred way to indicate a direct message for the
@@ -981,7 +432,7 @@ No changes; feature level used for Zulip 7.0 release.
   the modern convention with servers that support it, because support for
   `"private"` may eventually be removed.
 
-**Feature level 173**
+**Feature level 173**:
 
 * [`GET /scheduled_messages`](/api/get-scheduled-messages), [`DELETE
   /scheduled_messages/<int:scheduled_message_id>`](/api/delete-scheduled-message):
@@ -999,7 +450,7 @@ No changes; feature level used for Zulip 7.0 release.
   this endpoint now returns an error response
   (`"code": "MOVE_MESSAGES_TIME_LIMIT_EXCEEDED"`).
 
-**Feature level 171**
+**Feature level 171**:
 
 * [`POST /fetch_api_key`](/api/fetch-api-key),
   [`POST /dev_fetch_api_key`](/api/dev-fetch-api-key): The return values
@@ -1141,8 +592,8 @@ releases.
 
 **Feature level 157**
 
-* [`POST /invites`](/api/send-invites): Added support for invitations specifying
-  the empty list as the user's initial stream subscriptions. Previously, this
+* `POST /invites`: Added support for invitations specifying the empty
+  list as the user's initial stream subscriptions. Previously, this
   returned an error. This change was backported from the Zulip 7.0
   branch, and thus is available at feature levels 157-158 and 180+.
 
@@ -1400,9 +851,8 @@ user's profile.
 
 **Feature level 126**
 
-* [`POST /invites`](/api/send-invites),
-  [`POST /invites/multiuse`](/api/create-invite-link): Replaced
-  `invite_expires_in_days` parameter with `invite_expires_in_minutes`.
+* `POST /invites`, `POST /invites/multiuse`: Replaced `invite_expires_in_days`
+  parameter with `invite_expires_in_minutes`.
 
 **Feature level 125**
 
@@ -1461,10 +911,9 @@ No changes; feature level used for Zulip 5.0 release.
 
 **Feature level 117**
 
-* [`POST /invites`](/api/send-invites),
-  [`POST /invites/multiuse`](/api/create-invite-link): Added support
-  for passing `null` as the `invite_expires_in_days` parameter to
-  request an invitation that never expires.
+* `POST /invites`, `POST /invites/multiuse`: Added support for passing
+  `null` as the `invite_expires_in_days` parameter to request an
+  invitation that never expires.
 
 **Feature level 116**
 
@@ -1622,8 +1071,7 @@ No changes; feature level used for Zulip 5.0 release.
 
 * [`PATCH /realm/user_settings_defaults`](/api/update-realm-user-settings-defaults):
   Added new endpoint to update default values of user settings in a realm.
-* [`POST /invites`](/api/send-invites),
-  [`POST /invites/multiuse`](/api/create-invite-link): Added
+* `POST /invites`, `POST /invites/multiuse`: Added
   `invite_expires_in_days` parameter encoding the number days before
   the invitation should expire.
 
@@ -1798,17 +1246,11 @@ No changes; feature level used for Zulip 5.0 release.
 
 **Feature level 76**
 
-* [`POST /fetch_api_key`](/api/fetch-api-key),
-  [`POST /dev_fetch_api_key`](/api/dev-fetch-api-key): The HTTP status
-  for authentication errors is now 401. These previously used the HTTP
-  403 error status.
-* [Error handling](/api/rest-error-handling#common-error-responses): API
-  requests that involve a deactivated user or organization now use the
-  HTTP 401 error status. These previously used the HTTP 403 error status.
-* [Error handling](/api/rest-error-handling): All error responses
-  now include a `code` key with a machine-readable string value. The
-  default value for this key is `"BAD_REQUEST"` for general error
-  responses.
+* [`POST /fetch_api_key`](/api/fetch-api-key), [`POST
+  /dev_fetch_api_key`](/api/dev-fetch-api-key): The HTTP status for
+  authentication errors is now 401. This was previously 403.
+* All API endpoints now use the HTTP 401 error status for API requests
+  involving a deactivated user or realm. This was previously 403.
 * Mobile push notifications now include the `mentioned_user_group_id`
   and `mentioned_user_group_name` fields when a user group containing
   the user is mentioned.  Previously, these were indistinguishable
@@ -1890,9 +1332,8 @@ No changes; feature level used for Zulip 4.0 release.
 
 **Feature level 61**
 
-* [`POST /invites`](/api/send-invites),
-  [`POST /invites/multiuse`](/api/create-invite-link): Added support
-  for inviting users as moderators.
+* `POST /invites`, `POST /invites/multiuse`: Added support for
+  inviting users as moderators.
 
 **Feature level 60**
 
@@ -2074,9 +1515,8 @@ field with an integer field `invite_to_realm_policy`.
 
 * [`POST /users`](/api/create-user): Restricted access to organization
   administrators with the `can_create_users` permission.
-* [Error handling](/api/rest-error-handling#common-error-responses): The
-  `code` key will now be present in errors that are due to rate
-  limits, with a value of `"RATE_LIMIT_HIT"`.
+* [Error handling](/api/rest-error-handling): The `code` property will
+  now be present in errors due to rate limits.
 
 **Feature level 35**
 
@@ -2092,10 +1532,10 @@ field with an integer field `invite_to_realm_policy`.
 
 **Feature level 33**
 
-* [Markdown message formatting](/api/message-formatting#code-blocks):
-  [Code blocks](/help/code-blocks) now have a `data-code-language`
-  attribute attached to the outer HTML `div` element, recording the
-  programming language that was selected for syntax highlighting.
+* Markdown code blocks now have a `data-code-language` attribute
+  attached to the outer `div` element, recording the programming
+  language that was selecting for syntax highlighting.  This field
+  supports the upcoming "view in playground" feature for code blocks.
 
 **Feature level 32**
 
@@ -2151,9 +1591,9 @@ No changes; feature level used for Zulip 3.0 release.
 
 **Feature level 24**
 
-* [Markdown message formatting](/api/message-formatting#removed-features):
-  The rarely used `!avatar()` and `!gravatar()` markup syntax, which
-  was never documented and had inconsistent syntax, was removed.
+* The `!avatar()` and `!gravatar()` Markdown syntax, which was never
+  documented, had inconsistent syntax, and was rarely used, was
+  removed.
 
 **Feature level 23**
 
@@ -2171,8 +1611,8 @@ No changes; feature level used for Zulip 3.0 release.
   encoded as integers; (previously the implementation could send
   floats incorrectly suggesting that microsecond precision is
   relevant).
-* [`GET /invites`](/api/get-invites): Now encodes the user ID of the person
-   who created the invitation as `invited_by_user_id`, replacing the previous
+* `GET /invites`: Now encodes the user ID of the person who created
+   the invitation as `invited_by_user_id`, replacing the previous
    `ref` field (which had that user's Zulip display email address).
 * [`POST /register`](/api/register-queue): The encoding of an
   unlimited `realm_message_retention_days` in the response was changed
@@ -2218,8 +1658,8 @@ No changes; feature level used for Zulip 3.0 release.
 
 **Feature level 15**
 
-* [Markdown message formatting](/api/message-formatting#spoilers): Added
-  [spoilers](/help/spoilers) to supported message formatting features.
+* Added [spoilers](/help/format-your-message-using-markdown#spoilers)
+  to supported Markdown features.
 
 **Feature level 14**
 
@@ -2281,9 +1721,8 @@ No changes; feature level used for Zulip 3.0 release.
 * [`GET /users`](/api/get-users), [`GET /users/{user_id}`](/api/get-user)
   and [`GET /users/me`](/api/get-own-user): User objects now contain the
   `is_owner` field as well.
-* [Markdown message formatting](/api/message-formatting#global-times):
-  Added [global times](/help/global-times) to supported message
-  formatting features.
+* Added [time mentions](/help/format-your-message-using-markdown#mention-a-time)
+  to supported Markdown features.
 
 **Feature level 7**
 
@@ -2338,7 +1777,7 @@ No changes; feature level used for Zulip 3.0 release.
 * Added new `presence_enabled` user notification setting; previously
   [presence](/help/status-and-availability) was always enabled.
 
-**Feature level 2**
+**Feature level 2**:
 
 * [`POST /messages/{message_id}/reactions`](/api/add-reaction):
   The `reaction_type` parameter is optional; the server will guess the
@@ -2349,7 +1788,7 @@ No changes; feature level used for Zulip 3.0 release.
   `user_id` field.  The legacy `user` dictionary (which had
   inconsistent format between those two endpoints) is deprecated.
 
-**Feature level 1**
+**Feature level 1**:
 
 * [`PATCH /messages/{message_id}`](/api/update-message): Added the
   `stream_id` parameter to support moving messages between streams.
@@ -2385,7 +1824,7 @@ No changes; feature level used for Zulip 3.0 release.
   deprecating and replacing the `is_announcement_only` boolean.
 * [`GET /user_uploads/{realm_id_str}/{filename}`](/api/get-file-temporary-url):
   New endpoint added for requesting a temporary URL for an uploaded
-  file that does not require authentication to access (e.g., for passing
+  file that does not require authentication to access (e.g. for passing
   from a Zulip desktop, mobile, or terminal app to the user's default
   browser).
 * [`POST /register`](/api/register-queue), [`GET /events`](/api/get-events),
@@ -2409,7 +1848,7 @@ No changes; feature level used for Zulip 3.0 release.
 * [`POST /register`](/api/register-queue): Added
   `realm_default_external_accounts` to endpoint response.
 * [`GET /messages`](/api/get-messages): Added support for
-  [search/narrow options](/api/construct-narrow#changes) that use stream/user
+  [search/narrow options](/api/construct-narrow) that use stream/user
   IDs to specify a message's sender, its stream, and/or its recipient(s).
 * [`GET /users`](/api/get-users): Added `include_custom_profile_fields`
   to request custom profile field data.

api_docs/client-libraries.md

@@ -37,7 +37,7 @@ topic][integrations-thread] in
 or submit a pull request [updating this
 page](https://zulip.readthedocs.io/en/latest/documentation/api.html).
 
-[integrations-thread]: https://chat.zulip.org/#narrow/channel/127-integrations/topic/API.20client.20libraries/
+[integrations-thread]: https://chat.zulip.org/#narrow/stream/127-integrations/topic/API.20client.20libraries/
 
 ### Outdated
 

api_docs/configuring-python-bindings.md

@@ -5,65 +5,15 @@ easily, called the [Python bindings](https://pypi.python.org/pypi/zulip/).
 One of the most notable use cases for these bindings are bots developed
 using Zulip's [bot framework](/api/writing-bots).
 
-In order to use them, you need to configure them with your identity
-(account, API key, and Zulip server URL). There are a few ways to
-achieve that:
+In order to use them, you need to configure them with your API key and other
+settings. There are two ways to achieve that:
 
-- Using a `zuliprc` file, referenced via the `--config-file` option or
-  the `config_file` option to the `zulip.Client` constructor
-  (recommended for bots).
-- Using a `zuliprc` file in your home directory at `~/.zuliprc`
-  (recommended for your own API key).
-- Using the [environment
-  variables](https://en.wikipedia.org/wiki/Environment_variable)
-  documented below.
-- Using the `--api-key`, `--email`, and `--site` variables as command
-  line parameters.
-- Using the `api_key`, `email`, and `site` parameters to the
-  `zulip.Client` constructor.
+ - With a file called `.zuliprc`, located in your home directory.
+ - With
+   [environment variables](https://en.wikipedia.org/wiki/Environment_variable)
+   set up in your host machine.
 
-## Download a `zuliprc` file
-
-{start_tabs}
-
-{tab|for-a-bot}
-
-{settings_tab|your-bots}
-
-1. Click the **download** (<i class="fa fa-download"></i>) icon on the profile
-   card of the desired bot to download the bot's `zuliprc` file.
-
-!!! warn ""
-
-    Anyone with a bot's API key can impersonate the bot, so be careful with it!
-
-{tab|for-yourself}
-
-{settings_tab|account-and-privacy}
-
-1. Under **API key**, click **Manage your API key**.
-
-1. Enter your password, and click **Get API key**. If you don't know your
-   password, click **reset it** and follow the
-   instructions from there.
-
-1. Click **Download zuliprc** to download your `zuliprc` file.
-
-1. (optional) If you'd like your credentials to be used by default
-   when using the Zulip API on your computer, move the `zuliprc` file
-   to `~/.zuliprc` in your home directory.
-
-!!! warn ""
-
-    Anyone with your API key can impersonate you, so be doubly careful with it.
-
-{end_tabs}
-
-## Configuration keys and environment variables
-
-`zuliprc` is a configuration file written in the
-[INI file format](https://en.wikipedia.org/wiki/INI_file),
-which contains key-value pairs as shown in the following example:
+A `.zuliprc` file is a plain text document that looks like this:
 
 ```
 [api]
@@ -79,7 +29,7 @@ can be found in the following table:
 <table class="table">
     <thead>
         <tr>
-            <th><code>zuliprc</code> key</th>
+            <th><code>.zuliprc</code> key</th>
             <th>Environment variable</th>
             <th>Required</th>
             <th>Description</th>
@@ -152,10 +102,3 @@ can be found in the following table:
         </td>
     </tr>
 </table>
-
-## Related articles
-
-* [Installation instructions](/api/installation-instructions)
-* [API keys](/api/api-keys)
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)

api_docs/construct-narrow.md

@@ -1,18 +1,18 @@
 # Construct a narrow
 
 A **narrow** is a set of filters for Zulip messages, that can be based
-on many different factors (like sender, channel, topic, search
-keywords, etc.). Narrows are used in various places in the Zulip
+on many different factors (like sender, stream, topic, search
+keywords, etc.). Narrows are used in various places in the the Zulip
 API (most importantly, in the API for fetching messages).
 
 It is simplest to explain the algorithm for encoding a search as a
 narrow using a single example. Consider the following search query
 (written as it would be entered in the Zulip web app's search box).
-It filters for messages sent to channel `announce`, not sent by
+It filters for messages sent to stream `announce`, not sent by
 `iago@zulip.com`, and containing the words `cool` and `sunglasses`:
 
 ```
-channel:announce -sender:iago@zulip.com cool sunglasses
+stream:announce -sender:iago@zulip.com cool sunglasses
 ```
 
 This query would be JSON-encoded for use in the Zulip API using JSON
@@ -21,7 +21,7 @@ as a list of simple objects, as follows:
 ```json
 [
     {
-        "operator": "channel",
+        "operator": "stream",
         "operand": "announce"
     },
     {
@@ -40,82 +40,42 @@ The Zulip help center article on [searching for messages](/help/search-for-messa
 documents the majority of the search/narrow options supported by the
 Zulip API.
 
-Note that many narrows, including all that lack a `channel` or `channels`
+Note that many narrows, including all that lack a `stream` or `streams`
 operator, search the current user's personal message history. See
 [searching shared history](/help/search-for-messages#searching-shared-history)
 for details.
 
-Clients should note that the `is:unread` filter takes advantage of the
-fact that there is a database index for unread messages, which can be an
-important optimization when fetching messages in certain cases (e.g.,
-when [adding the `read` flag to a user's personal
-messages](/api/update-message-flags-for-narrow)).
+**Changes**: In Zulip 7.0 (feature level 177), support was added
+for three filters related to direct messages: `is:dm`, `dm` and
+`dm-including`. The `dm` operator replaced and deprecated the
+`pm-with` operator. The `is:dm` filter replaced and deprecated
+the `is:private` filter. The `dm-including` operator replaced and
+deprecated the `group-pm-with` operator.
 
-## Changes
+The `dm-including` and `group-pm-with` operators return slightly
+different results. For example, `dm-including:1234` returns all
+direct messages (1-on-1 and group) that include the current user
+and the user with the unique user ID of `1234`. On the other hand,
+`group-pm-with:1234` returned only group direct messages that included
+the current user and the user with the unique user ID of `1234`.
 
-* In Zulip 9.0 (feature level 271), support was added for a new filter
-  operator, `with`, which uses a [message ID](#message-ids) for its
-  operand, and is designed for creating permanent links to topics.
-
-* In Zulip 9.0 (feature level 265), support was added for a new
-  `is:followed` filter, matching messages in topics that the current
-  user is [following](/help/follow-a-topic).
-
-* In Zulip 9.0 (feature level 250), support was added for two filters
-  related to stream messages: `channel` and `channels`. The `channel`
-  operator is an alias for the `stream` operator. The `channels`
-  operator is an alias for the `streams` operator. Both `channel` and
-  `channels` return the same exact results as `stream` and `streams`
-  respectively.
-
-* In Zulip 9.0 (feature level 249), support was added for a new filter,
-  `has:reaction`, which returns messages that have at least one [emoji
-  reaction](/help/emoji-reactions).
-
-* In Zulip 7.0 (feature level 177), support was added for three filters
-  related to direct messages: `is:dm`, `dm` and `dm-including`. The
-  `dm` operator replaced and deprecated the `pm-with` operator. The
-  `is:dm` filter replaced and deprecated the `is:private` filter. The
-  `dm-including` operator replaced and deprecated the `group-pm-with`
-  operator.
-
-    * The `dm-including` and `group-pm-with` operators return slightly
-      different results. For example, `dm-including:1234` returns all
-      direct messages (1-on-1 and group) that include the current user
-      and the user with the unique user ID of `1234`. On the other hand,
-      `group-pm-with:1234` returned only group direct messages that
-      included the current user and the user with the unique user ID of
-      `1234`.
-
-    * Both `dm` and `is:dm` are aliases of `pm-with` and `is:private`
-      respectively, and return the same exact results that the
-      deprecated filters did.
+Both `dm` and `is:dm` are aliases of `pm-with` and `is:private`
+respectively, and return the same exact results that the deprecated
+filters did.
 
 ## Narrows that use IDs
 
 ### Message IDs
 
-The `near`, `id` and `with` operators use message IDs for their
-operands. The `near` and `id` operators are documented in the help
-center [here](/help/search-for-messages#search-by-message-id).
+The `near` and `id` operators, documented in the help center, use message
+IDs for their operands.
 
-The `with` operator is designed to be used for permanent links to topics,
-which means they should continue to work when the topic is
-[moved](/help/move-content-to-another-topic) or
-[resolved](/help/resolve-a-topic). If the message with the specified ID
-exists, and can be accessed by the user, then it will return messages
-with the `channel`/`topic`/`dm` operators corresponding to the current
-conversation containing that message, and replacing any such filters
-included in the narrow.
-
-* `with:12345`: Search for the conversation that contains the message
-  with ID `12345`.
 * `near:12345`: Search messages around the message with ID `12345`.
-* `id:12345`: Search for only the message with ID `12345`.
+* `id:12345`: Search for only message with ID `12345`.
 
-The message ID operand for the `with` and `id` operators may be encoded
-as either a number or a string. The message ID operand for the `near`
-operator must be encoded as a string.
+The message ID operand for the `id` operator may be encoded as either a
+number or a string. The message ID operand for the `near` operator must
+be encoded as a string.
 
 **Changes**: Prior to Zulip 8.0 (feature level 194), the message ID
 operand for the `id` operator needed to be encoded as a string.
@@ -130,13 +90,13 @@ operand for the `id` operator needed to be encoded as a string.
 ]
 ```
 
-### Channel and user IDs
+### Stream and user IDs
 
 There are a few additional narrow/search options (new in Zulip 2.1)
-that use either channel IDs or user IDs that are not documented in the
+that use either stream IDs or user IDs that are not documented in the
 help center because they are primarily useful to API clients:
 
-* `channel:1234`: Search messages sent to the channel with ID `1234`.
+* `stream:1234`: Search messages sent to the stream with ID `1234`.
 * `sender:1234`: Search messages sent by user ID `1234`.
 * `dm:1234`: Search the direct message conversation between
   you and user ID `1234`.
@@ -145,12 +105,6 @@ help center because they are primarily useful to API clients:
 * `dm-including:1234`: Search all direct messages (1-on-1 and group)
   that include you and user ID `1234`.
 
-!!! tip ""
-
-    A user ID can be found by [viewing a user's profile][view-profile]
-    in the web or desktop apps. A channel ID can be found when [browsing
-    channels][browse-channels] in the web or desktop apps.
-
 The operands for these search options must be encoded either as an
 integer ID or a JSON list of integer IDs. For example, to query
 messages sent by a user 1234 to a direct message thread with yourself,
@@ -168,6 +122,3 @@ user 1234, and user 5678, the correct JSON-encoded query is:
     }
 ]
 ```
-
-[view-profile]: /help/view-someones-profile
-[browse-channels]: /help/introduction-to-channels#browse-and-subscribe-to-channels

api_docs/create-scheduled-message.md

@@ -11,7 +11,7 @@
 {tab|curl}
 
 ``` curl
-# Create a scheduled channel message
+# Create a scheduled stream message
 curl -X POST {{ api_url }}/v1/scheduled_messages \
     -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
     --data-urlencode type=stream \

api_docs/create-stream.md

@@ -1,6 +1,6 @@
-# Create a channel
+# Create a stream
 
-You can create a channel using Zulip's REST API by submitting a
-[subscribe](/api/subscribe) request with a channel name that
+You can create a stream using Zulip's REST API by submitting a
+[subscribe](/api/subscribe) request with a stream name that
 doesn't yet exist and passing appropriate parameters to define
-the initial configuration of the new channel.
+the initial configuration of the new stream.

api_docs/deploying-bots.md

@@ -44,7 +44,7 @@ Botserver interaction are:
 
 1. The Zulip server sends a POST request to the Botserver on `https://bot-server.example.com/`:
 
-    ```
+    ```json
     {
       "message":{
         "content":"@**My Bot User** hello world",
@@ -57,8 +57,9 @@ Botserver interaction are:
 
     This URL is configured in the Zulip web-app in your Bot User's settings.
 
-1. The Botserver searches for a bot to handle the message, and executes your
-   bot's `handle_message` code.
+1. The Botserver searches for a bot to handle the message.
+
+1. The Botserver executes your bot's `handle_message` code.
 
 Your bot's code should work just like it does with `zulip-run-bot`;
 for example, you reply using
@@ -74,7 +75,6 @@ pip3 install zulip_botserver
 
 ### Running a bot using the Zulip Botserver
 
-{start_tabs}
 
 1. Construct the URL for your bot, which will be of the form:
 
@@ -89,7 +89,7 @@ pip3 install zulip_botserver
 1. Register new bot users on the Zulip server's web interface.
 
     * Log in to the Zulip server.
-    * Navigate to *Personal settings (<i class="zulip-icon zulip-icon-gear"></i>)* -> *Bots* -> *Add a new bot*.
+    * Navigate to *Personal settings (<i class="fa fa-cog"></i>)* -> *Bots* -> *Add a new bot*.
       Select *Outgoing webhook* for bot type, fill out the form (using
       the URL from above) and click on *Create bot*.
     * A new bot user should appear in the *Active bots* panel.
@@ -108,15 +108,11 @@ pip3 install zulip_botserver
 1.  Congrats, everything is set up! Test your Botserver like you would
     test a normal bot.
 
-{end_tabs}
-
 ### Running multiple bots using the Zulip Botserver
 
 The Zulip Botserver also supports running multiple bots from a single
 Botserver process.  You can do this with the following procedure.
 
-{start_tabs}
-
 1. Download the `botserverrc` from the `your-bots` settings page, using
    the "Download config of all active outgoing webhook bots in Zulip
    Botserver format." option at the top.
@@ -164,8 +160,6 @@ Botserver process.  You can do this with the following procedure.
 
      If omitted, `hostname` defaults to `127.0.0.1` and `port` to `5002`.
 
-{end_tabs}
-
 ### Running Zulip Botserver with supervisord
 
 [supervisord](http://supervisord.org/) is a popular tool for running
@@ -176,9 +170,7 @@ section documents how to run the Zulip Botserver using *supervisord*.
 Running the Zulip Botserver with *supervisord* works almost like
 running it manually.
 
-{start_tabs}
-
-1.  Install *supervisord* via your package manager; e.g., on Debian/Ubuntu:
+1.  Install *supervisord* via your package manager; e.g. on Debian/Ubuntu:
 
      ```
      sudo apt-get install supervisor
@@ -224,8 +216,6 @@ running it manually.
     The standard output of the Botserver will be logged to the path in
     your *supervisord* configuration.
 
-{end_tabs}
-
 If you are hosting the Botserver yourself (as opposed to using a
 hosting service that provides SSL), we recommend securing your
 Botserver with SSL using an `nginx` or `Apache` reverse proxy and
@@ -233,17 +223,18 @@ Botserver with SSL using an `nginx` or `Apache` reverse proxy and
 
 ### Troubleshooting
 
-- Make sure the API key you're using is for an [outgoing webhook
-  bot](/api/outgoing-webhooks) and you've
-  correctly configured the URL for your Botserver.
+1. Make sure the API key you're using is for an [outgoing webhook
+   bot](/api/outgoing-webhooks) and you've
+   correctly configured the URL for your Botserver.
 
-- Your Botserver needs to be accessible from your Zulip server over
-  HTTP(S).  Make sure any firewall allows the connection.  We
-  recommend using [zulip-run-bot](running-bots) instead for
-  development/testing on a laptop or other non-server system.
-  If your Zulip server is self-hosted, you can test by running `curl
-  http://zulipbotserver.example.com:5002` from your Zulip server;
-  the output should be:
+1.  Your Botserver needs to be accessible from your Zulip server over
+    HTTP(S).  Make sure any firewall allows the connection.  We
+    recommend using [zulip-run-bot](running-bots) instead for
+    development/testing on a laptop or other non-server system.
+
+    If your Zulip server is self-hosted, you can test by running `curl
+    http://zulipbotserver.example.com:5002` from your Zulip server;
+    the output should be:
 
     ```
     $ curl http://zulipbotserver.example.com:5002/
@@ -252,9 +243,3 @@ Botserver with SSL using an `nginx` or `Apache` reverse proxy and
     <h1>Method Not Allowed</h1>
     <p>The method is not allowed for the requested URL.</p>
     ```
-
-## Related articles
-
-* [Non-webhook integrations](/api/non-webhook-integrations)
-* [Running bots](/api/running-bots)
-* [Writing bots](/api/writing-bots)

api_docs/group-setting-values.md

@@ -1,90 +0,0 @@
-# Group-setting values
-
-Settings defining permissions in Zulip are increasingly represented
-using [user groups](/help/user-groups), which offer much more flexible
-configuration than the older [roles](/api/roles-and-permissions) system.
-
-!!! warn ""
-
-    This API feature is under development, and currently only values that
-    correspond to a single named user group are permitted in
-    production environments, pending the web application UI supporting
-    displaying more complex values correctly.
-
-In the API, these settings are represented using a **group-setting
-value**, which can take two forms:
-
-- An integer user group ID, which can be either a named user group
-  visible in the UI or a [role-based system group](#system-groups).
-- An object with fields `direct_member_ids` containing a list of
-  integer user IDs and `direct_subgroup_ids` containing a list of
-  integer group IDs. The setting's value is the union of the
-  identified collection of users and groups.
-
-Group-setting values in the object form function very much like a
-formal user group object, without requiring the naming and UI clutter
-overhead involved with creating a visible user group just to store the
-value of a single setting.
-
-The server will canonicalize an object with empty `direct_member_ids`
-and with `direct_subgroup_ids` containing just the given group ID to
-the integer format.
-
-## System groups
-
-The Zulip server maintains a collection of system groups that
-correspond to the users with a given role; this makes it convenient to
-store concepts like "all administrators" in a group-setting
-value. These use a special naming convention and can be recognized by
-the `is_system_group` property on their group object.
-
-The following system groups are maintained by the Zulip server:
-
-- `role:internet`: Everyone on the Internet has this permission; this
-  is used to configure the [public access
-  option](/help/public-access-option).
-- `role:everyone`: All users, including guests.
-- `role:members`: All users, excluding guests.
-- `role:fullmembers`: All [full
-  members](https://zulip.com/api/roles-and-permissions#determining-if-a-user-is-a-full-member)
-  of the organization.
-- `role:moderators`: All users with at least the moderator role.
-- `role:administrators`: All users with at least the administrator
-  role.
-- `role:owners`: All users with the owner role.
-- `role:nobody`: The formal empty group. Used in the API to represent
-  disabling a feature.
-
-Client UI for setting a permission is encouraged to display system
-groups using their description, rather than using their names, which
-are chosen to be unique and clear in the API.
-
-System groups should generally not be displayed in UI for
-administering an organization's user groups, since they are not
-directly mutable.
-
-## Updating group-setting values
-
-The Zulip API uses a special format for modifying an existing setting
-using a group-setting value.
-
-A **group-setting update** is an object with a `new` field and an
-optional `old` field, each containing a group-setting value. The
-setting's value will be set to the membership expressed by the `new`
-field.
-
-The `old` field expresses the client's understanding of the current
-value of the setting. If the `old` field is present and does not match
-the actual current value of the setting, then the request will fail
-with error code `EXPECTATION_MISMATCH` and no changes will be applied.
-
-When a user edits the setting in a UI, the resulting API request
-should generally always include the `old` field, giving the value
-the list had when the user started editing. This accurately expresses
-the user's intent, and if two users edit the same list around the
-same time, it prevents a situation where the second change
-accidentally reverts the first one without either user noticing.
-
-Omitting `old` is appropriate where the intent really is a new complete
-list rather than an edit, for example in an integration that syncs the
-list from an external source of truth.

api_docs/http-headers.md

@@ -49,7 +49,7 @@ client = zulip.Client(
 If you are working on an integration that you plan to share outside
 your organization, you can get help picking a good name in
 `#integrations` in the [Zulip development
-community](https://zulip.com/development-community/).
+community](https://zulip.com/development-community).
 
 ## Rate-limiting response headers
 

api_docs/include/rest-endpoints.md

@@ -10,12 +10,12 @@
 * [Remove an emoji reaction](/api/remove-reaction)
 * [Render a message](/api/render-message)
 * [Fetch a single message](/api/get-message)
-* [Check if messages match a narrow](/api/check-messages-match-narrow)
+* [Check if messages match narrow](/api/check-messages-match-narrow)
 * [Get a message's edit history](/api/get-message-history)
 * [Update personal message flags](/api/update-message-flags)
 * [Update personal message flags for narrow](/api/update-message-flags-for-narrow)
 * [Mark all messages as read](/api/mark-all-as-read)
-* [Mark messages in a channel as read](/api/mark-stream-as-read)
+* [Mark messages in a stream as read](/api/mark-stream-as-read)
 * [Mark messages in a topic as read](/api/mark-topic-as-read)
 * [Get a message's read receipts](/api/get-read-receipts)
 
@@ -33,45 +33,42 @@
 * [Edit a draft](/api/edit-draft)
 * [Delete a draft](/api/delete-draft)
 
-#### Channels
+#### Streams
 
-* [Get subscribed channels](/api/get-subscriptions)
-* [Subscribe to a channel](/api/subscribe)
-* [Unsubscribe from a channel](/api/unsubscribe)
+* [Get subscribed streams](/api/get-subscriptions)
+* [Subscribe to a stream](/api/subscribe)
+* [Unsubscribe from a stream](/api/unsubscribe)
 * [Get subscription status](/api/get-subscription-status)
-* [Get channel subscribers](/api/get-subscribers)
+* [Get all subscribers](/api/get-subscribers)
 * [Update subscription settings](/api/update-subscription-settings)
-* [Get all channels](/api/get-streams)
-* [Get a channel by ID](/api/get-stream-by-id)
-* [Get channel ID](/api/get-stream-id)
-* [Create a channel](/api/create-stream)
-* [Update a channel](/api/update-stream)
-* [Archive a channel](/api/archive-stream)
-* [Get channel's email address](/api/get-stream-email-address)
-* [Get topics in a channel](/api/get-stream-topics)
+* [Get all streams](/api/get-streams)
+* [Get a stream by ID](/api/get-stream-by-id)
+* [Get stream ID](/api/get-stream-id)
+* [Create a stream](/api/create-stream)
+* [Update a stream](/api/update-stream)
+* [Archive a stream](/api/archive-stream)
+* [Get topics in a stream](/api/get-stream-topics)
 * [Topic muting](/api/mute-topic)
 * [Update personal preferences for a topic](/api/update-user-topic)
 * [Delete a topic](/api/delete-topic)
-* [Add a default channel](/api/add-default-stream)
-* [Remove a default channel](/api/remove-default-stream)
+* [Add a default stream](/api/add-default-stream)
+* [Remove a default stream](/api/remove-default-stream)
 
 #### Users
 
+* [Get all users](/api/get-users)
+* [Get own user](/api/get-own-user)
 * [Get a user](/api/get-user)
 * [Get a user by email](/api/get-user-by-email)
-* [Get own user](/api/get-own-user)
-* [Get all users](/api/get-users)
-* [Create a user](/api/create-user)
 * [Update a user](/api/update-user)
-* [Deactivate a user](/api/deactivate-user)
-* [Deactivate own user](/api/deactivate-own-user)
-* [Reactivate a user](/api/reactivate-user)
-* [Get a user's status](/api/get-user-status)
 * [Update your status](/api/update-status)
+* [Create a user](/api/create-user)
+* [Deactivate a user](/api/deactivate-user)
+* [Reactivate a user](/api/reactivate-user)
+* [Deactivate own user](/api/deactivate-own-user)
 * [Set "typing" status](/api/set-typing-status)
-* [Get a user's presence](/api/get-user-presence)
+* [Get user presence](/api/get-user-presence)
 * [Get presence of all users](/api/get-presence)
-* [Update your presence](/api/update-presence)
 * [Get attachments](/api/get-attachments)
 * [Delete an attachment](/api/remove-attachment)
 * [Update settings](/api/update-settings)
@@ -80,25 +77,16 @@
 * [Update a user group](/api/update-user-group)
 * [Delete a user group](/api/remove-user-group)
 * [Update user group members](/api/update-user-group-members)
-* [Update subgroups of a user group](/api/update-user-group-subgroups)
+* [Update user group subgroups](/api/update-user-group-subgroups)
 * [Get user group membership status](/api/get-is-user-group-member)
 * [Get user group members](/api/get-user-group-members)
-* [Get subgroups of a user group](/api/get-user-group-subgroups)
+* [Get subgroups of user group](/api/get-user-group-subgroups)
 * [Mute a user](/api/mute-user)
 * [Unmute a user](/api/unmute-user)
 * [Get all alert words](/api/get-alert-words)
 * [Add alert words](/api/add-alert-words)
 * [Remove alert words](/api/remove-alert-words)
 
-#### Invitations
-
-* [Get all invitations](/api/get-invites)
-* [Send invitations](/api/send-invites)
-* [Create a reusable invitation link](/api/create-invite-link)
-* [Resend an email invitation](/api/resend-email-invite)
-* [Revoke an email invitation](/api/revoke-email-invite)
-* [Revoke a reusable invitation link](/api/revoke-invite-link)
-
 #### Server & organizations
 
 * [Get server settings](/api/get-server-settings)
@@ -128,9 +116,3 @@
 
 * [Fetch an API key (production)](/api/fetch-api-key)
 * [Fetch an API key (development only)](/api/dev-fetch-api-key)
-* [Send a test notification to mobile device(s)](/api/test-notify)
-* [Add an APNs device token](/api/add-apns-token)
-* [Remove an APNs device token](/api/remove-apns-token)
-* [Add an FCM registration token](/api/add-fcm-token)
-* [Remove an FCM registration token](/api/remove-fcm-token)
-* [Create BigBlueButton video call](/api/create-big-blue-button-video-call)

api_docs/incoming-webhooks-overview.md

@@ -1,7 +1,7 @@
 # Incoming webhook integrations
 
 An incoming webhook allows a third-party service to push data to Zulip when
-something happens. There are several ways to set up an incoming webhook in
+something happens.  There's several ways to do an incoming webhook in
 Zulip:
 
 * Use our [REST API](/api/rest) endpoint for [sending
@@ -11,9 +11,9 @@ Zulip:
 * Use one of our supported [integration
   frameworks](/integrations/meta-integration), such as the
   [Slack-compatible incoming webhook](/integrations/doc/slack_incoming),
-  [Zapier integration](/integrations/doc/zapier), or
+  [Zapier integration](/integrations/docs/zapier), or
   [IFTTT integration](/integrations/doc/ifttt).
-* Implementing an incoming webhook integration (detailed on this page),
+* Adding an incoming webhook integration (detailed on this page),
   where all the logic for formatting the Zulip messages lives in the
   Zulip server.  This is how most of [Zulip's official
   integrations](/integrations/) work, because they enable Zulip to
@@ -22,7 +22,7 @@ Zulip:
   Zulip).
 
 In an incoming webhook integration, the third-party service's
-"outgoing webhook" feature sends an `HTTP POST` to a special URL when
+"outgoing webhook" feature sends an `HTTP POST`s to a special URL when
 it has something for you, and then the Zulip "incoming webhook"
 integration handles that incoming data to format and send a message in
 Zulip.
@@ -40,18 +40,18 @@ process.
   <https://webhook.site/>, or a similar site to capture an example
   webhook payload from the third-party service. Create a
   `zerver/webhooks/<mywebhook>/fixtures/` directory, and add the
-  captured JSON payload as a test fixture.
+  captured payload as a test fixture.
 
-* Create an `Integration` object, and add it to the `WEBHOOK_INTEGRATIONS`
-  list in `zerver/lib/integrations.py`. Search for `WebhookIntegration` in that
-  file to find an existing one to copy.
+* Create an `Integration` object, and add it to `WEBHOOK_INTEGRATIONS` in
+  `zerver/lib/integrations.py`. Search for `webhook` in that file to find an
+  existing one to copy.
 
-* Write a draft webhook handler in `zerver/webhooks/<mywebhook>/view.py`. There
-  are a lot of examples in the `zerver/webhooks/` directory that you can copy.
-  We recommend templating from a short one, like `zendesk`.
+* Write a draft webhook handler under `zerver/webhooks/`. There are a lot of
+  examples in that directory that you can copy. We recommend templating off
+  a short one, like `zendesk`.
 
-* Write a test for your fixture in `zerver/webhooks/<mywebhook>/tests.py`.
-  Run the test for your integration like this:
+* Add a test for your fixture at `zerver/webhooks/<mywebhook>/tests.py`.
+  Run the tests for your integration like this:
 
     ```
     tools/test-backend zerver/webhooks/<mywebhook>/
@@ -64,10 +64,10 @@ process.
   service will make, and add tests for them; usually this part of the
   process is pretty fast.
 
-* Document the integration in `zerver/webhooks/<mywebhook>/doc.md`(required for
-  getting it merged into Zulip). You can use existing documentation, like
-  [this one](https://raw.githubusercontent.com/zulip/zulip/main/zerver/webhooks/github/doc.md),
-  as a template. This should not take more than 15 minutes, even if you don't speak English
+* Document the integration (required for getting it merged into Zulip). You
+  can template off an existing guide, like
+  [this one](https://raw.githubusercontent.com/zulip/zulip/main/zerver/webhooks/github/doc.md).
+  This should not take more than 15 minutes, even if you don't speak English
   as a first language (we'll clean up the text before merging).
 
 ## Hello world walkthrough
@@ -84,9 +84,9 @@ below are for a webhook named `MyWebHook`.
 
 * `zerver/webhooks/mywebhook/__init__.py`: Empty file that is an obligatory
    part of every python package.  Remember to `git add` it.
-* `zerver/webhooks/mywebhook/view.py`: The main webhook integration function,
-  called `api_mywebhook_webhook`, along with any necessary helper functions.
-* `zerver/webhooks/mywebhook/fixtures/message_type.json`: Sample JSON payload data
+* `zerver/webhooks/mywebhook/view.py`: The main webhook integration function
+  as well as any needed helper functions.
+* `zerver/webhooks/mywebhook/fixtures/messagetype.json`: Sample json payload data
   used by tests. Add one fixture file per type of message supported by your
   integration.
 * `zerver/webhooks/mywebhook/tests.py`: Tests for your webhook.
@@ -95,9 +95,9 @@ below are for a webhook named `MyWebHook`.
 * `static/images/integrations/logos/mywebhook.svg`: A square logo for the
   platform/server/product you are integrating. Used on the documentation
   pages as well as the sender's avatar for messages sent by the integration.
-* `static/images/integrations/mywebhook/001.png`: A screenshot of a message
+* `static/images/integrations/mywebhook/001.svg`: A screenshot of a message
   sent by the integration, used on the documentation page. This can be
-  generated by running `tools/screenshots/generate-integration-docs-screenshot --integration mywebhook`.
+  generated by running `tools/generate-integration-docs-screenshot --integration mywebhook`.
 * `static/images/integrations/bot_avatars/mywebhook.png`: A square logo for the
   platform/server/product you are integrating which is used to create the avatar
   for generating screenshots with. This can be generated automatically from
@@ -113,7 +113,7 @@ below are for a webhook named `MyWebHook`.
   `zerver/webhooks/mywebhook/view.py`. Also add your integration to
   `DOC_SCREENSHOT_CONFIG`. This will allow you to automatically generate
   a screenshot for the documentation by running
-  `tools/screenshots/generate-integration-docs-screenshot --integration mywebhook`.
+  `tools/generate-integration-docs-screenshot --integration mywebhook`.
 
 ## Common Helpers
 
@@ -125,19 +125,19 @@ below are for a webhook named `MyWebHook`.
 ## General advice
 
 * Consider using our Zulip markup to make the output from your
-  integration especially attractive or useful (e.g., emoji, Markdown
-  emphasis, or @-mentions).
+  integration especially attractive or useful (e.g.  emoji, Markdown
+  emphasis or @-mentions).
 
 * Use topics effectively to ensure sequential messages about the same
   thing are threaded together; this makes for much better consumption
-  by users.  E.g., for a bug tracker integration, put the bug number in
+  by users.  E.g. for a bug tracker integration, put the bug number in
   the topic for all messages; for an integration like Nagios, put the
   service in the topic.
 
 * Integrations that don't match a team's workflow can often be
   uselessly spammy.  Give careful thought to providing options for
   triggering Zulip messages only for certain message types, certain
-  projects, or sending different messages to different channels/topics,
+  projects, or sending different messages to different streams/topics,
   to make it easy for teams to configure the integration to support
   their workflow.
 
@@ -155,69 +155,3 @@ below are for a webhook named `MyWebHook`.
   testing with live data from the service you're integrating and can help you
   spot why something isn't working or if the service is using custom HTTP
   headers.
-
-## URL specification
-
-The base URL for an incoming webhook integration bot, where
-`INTEGRATION_NAME` is the name of the specific webhook integration and
-`API_KEY` is the API key of the bot created by the user for the
-integration, is:
-
-```
-{{ api_url }}/v1/external/INTEGRATION_NAME?api_key=API_KEY
-```
-
-The list of existing webhook integrations can be found by browsing the
-[Integrations documentation](/integrations/) or in
-`zerver/lib/integrations.py` at `WEBHOOK_INTEGRATIONS`.
-
-Parameters accepted in the URL include:
-
-### api_key *(required)*
-
-The API key of the bot created by the user for the integration. To get a
-bot's API key, see the [API keys](/api/api-keys) documentation.
-
-### stream
-
-The channel for the integration to send notifications to. Can be either
-the channel ID or the [URL-encoded][url-encoder] channel name. By default
-the integration will send direct messages to the bot's owner.
-
-!!! tip ""
-
-    A channel ID can be found when [browsing channels][browse-channels]
-    in the web or desktop apps.
-
-### topic
-
-The topic in the specified channel for the integration to send
-notifications to. The topic should also be [URL-encoded][url-encoder].
-By default the integration will have a topic configured for channel
-messages.
-
-### only_events, exclude_events
-
-Some incoming webhook integrations support these parameters to filter
-which events will trigger a notification. You can append either
-`&only_events=["event_a","event_b"]` or
-`&exclude_events=["event_a","event_b"]` (or both, with different events)
-to the URL, with an arbitrary number of supported events.
-
-You can use UNIX-style wildcards like `*` to include multiple events.
-For example, `test*` matches every event that starts with `test`.
-
-!!! tip ""
-
-    For a list of supported events, see a specific [integration's
-    documentation](/integrations) page.
-
-[browse-channels]: /help/introduction-to-channels#browse-and-subscribe-to-channels
-[add-bot]: /help/add-a-bot-or-integration
-[url-encoder]: https://www.urlencoder.org/
-
-## Related articles
-
-* [Integrations overview](/api/integrations-overview)
-* [Incoming webhook walkthrough](/api/incoming-webhooks-walkthrough)
-* [Non-webhook integrations](/api/non-webhook-integrations)

api_docs/incoming-webhooks-walkthrough.md

@@ -2,7 +2,7 @@
 
 Below, we explain each part of a simple incoming webhook integration,
 called **Hello World**.  This integration sends a "hello" message to the `test`
-channel and includes a link to the Wikipedia article of the day, which
+stream and includes a link to the Wikipedia article of the day, which
 it formats from json data it receives in the http request.
 
 Use this walkthrough to learn how to write your first webhook
@@ -69,10 +69,10 @@ integration uses.
 ## Step 1: Initialize your webhook python package
 
 In the `zerver/webhooks/` directory, create new subdirectory that will
-contain all of the corresponding code. In our example, it will be
+contain all of corresponding code.  In our example it will be
 `helloworld`. The new directory will be a python package, so you have
-to create an empty `__init__.py` file in that directory via, for
-example, `touch zerver/webhooks/helloworld/__init__.py`.
+to create an empty `__init__.py` file in that directory via e.g.
+`touch zerver/webhooks/helloworld/__init__.py`.
 
 ## Step 2: Create main webhook code
 
@@ -82,24 +82,25 @@ python file, `zerver/webhooks/mywebhook/view.py`.
 The Hello World integration is in `zerver/webhooks/helloworld/view.py`:
 
 ```python
+from typing import Any, Dict, Sequence
+
 from django.http import HttpRequest, HttpResponse
 
 from zerver.decorator import webhook_view
+from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
-from zerver.lib.typed_endpoint import JsonBodyPayload, typed_endpoint
-from zerver.lib.validator import WildValue, check_string
 from zerver.lib.webhooks.common import check_send_webhook_message
 from zerver.models import UserProfile
 
 
 @webhook_view("HelloWorld")
-@typed_endpoint
+@has_request_variables
 def api_helloworld_webhook(
     request: HttpRequest,
     user_profile: UserProfile,
-    *,
-    payload: JsonBodyPayload[WildValue],
+    payload: Dict[str, Sequence[Dict[str, Any]]] = REQ(argument_type="body"),
 ) -> HttpResponse:
+
     # construct the body of the message
     body = "Hello! I am happy to be here! :smile:"
 
@@ -107,10 +108,7 @@ def api_helloworld_webhook(
     body_template = (
         "\nThe Wikipedia featured article for today is **[{featured_title}]({featured_url})**"
     )
-    body += body_template.format(
-        featured_title=payload["featured_title"].tame(check_string),
-        featured_url=payload["featured_url"].tame(check_string),
-    )
+    body += body_template.format(**payload)
 
     topic = "Hello World"
 
@@ -122,13 +120,14 @@ def api_helloworld_webhook(
 
 The above code imports the required functions and defines the main webhook
 function `api_helloworld_webhook`, decorating it with `webhook_view` and
-`typed_endpoint`. The `typed_endpoint` decorator allows you to
-access request variables with `JsonBodyPayload()`. You can find more about `JsonBodyPayload` and request variables in [Writing views](
+`has_request_variables`. The `has_request_variables` decorator allows you to
+access request variables with `REQ()`. You can find more about `REQ` and request
+variables in [Writing views](
 https://zulip.readthedocs.io/en/latest/tutorials/writing-views.html#request-variables).
 
 You must pass the name of your integration to the
 `webhook_view` decorator; that name will be used to
-describe your integration in Zulip's analytics (e.g., the `/stats`
+describe your integration in Zulip's analytics (e.g. the `/stats`
 page). Here we have used `HelloWorld`. To be consistent with other
 integrations, use the name of the product you are integrating in camel
 case, spelled as the product spells its own name (except always first
@@ -144,14 +143,14 @@ You should name your webhook function as such
 integration and is always lower-case.
 
 At minimum, the webhook function must accept `request` (Django
-[HttpRequest](https://docs.djangoproject.com/en/5.0/ref/request-response/#django.http.HttpRequest)
+[HttpRequest](https://docs.djangoproject.com/en/3.2/ref/request-response/#django.http.HttpRequest)
 object), and `user_profile` (Zulip's user object). You may also want to
 define additional parameters using the `REQ` object.
 
 In the example above, we have defined `payload` which is populated
 from the body of the http request, `stream` with a default of `test`
 (available by default in the Zulip development environment), and
-`topic` with a default of `Hello World`. If your webhook uses a custom channel,
+`topic` with a default of `Hello World`. If your webhook uses a custom stream,
 it must exist before a message can be created in it. (See
 [Step 4: Create automated tests](#step-5-create-automated-tests) for how to handle this in tests.)
 
@@ -170,7 +169,7 @@ link to the Wikipedia article of the day as provided by the json payload.
 Then we send a message with `check_send_webhook_message`, which will
 validate the message and do the following:
 
-* Send a public (channel) message if the `stream` query parameter is
+* Send a public (stream) message if the `stream` query parameter is
   specified in the webhook URL.
 * If the `stream` query parameter isn't specified, it will send a direct
   message to the owner of the webhook bot.
@@ -192,7 +191,7 @@ WEBHOOK_INTEGRATIONS: List[WebhookIntegration] = [
 And you'll find the entry for Hello World:
 
 ```python
-  WebhookIntegration("helloworld", ["misc"], display_name="Hello World"),
+  WebhookIntegration('helloworld', ['misc'], display_name='Hello World'),
 ```
 
 This tells the Zulip API to call the `api_helloworld_webhook` function in
@@ -200,7 +199,7 @@ This tells the Zulip API to call the `api_helloworld_webhook` function in
 `/api/v1/external/helloworld`.
 
 This line also tells Zulip to generate an entry for Hello World on the Zulip
-integrations page using `static/images/integrations/logos/helloworld.svg` as its
+integrations page using `static/images/integrations/logos/helloworld.png` as its
 icon. The second positional argument defines a list of categories for the
 integration.
 
@@ -289,19 +288,15 @@ the [management commands][management-commands] documentation.
 ### Integrations Dev Panel
 This is the GUI tool.
 
-{start_tabs}
-
 1. Run `./tools/run-dev` then go to http://localhost:9991/devtools/integrations/.
 
-1. Set the following mandatory fields:
+2. Set the following mandatory fields:
 **Bot** - Any incoming webhook bot.
 **Integration** - One of the integrations.
 **Fixture** - Though not mandatory, it's recommended that you select one and then tweak it if necessary.
 The remaining fields are optional, and the URL will automatically be generated.
 
-1. Click **Send**!
-
-{end_tabs}
+3. Click **Send**!
 
 By opening Zulip in one tab and then this tool in another, you can quickly tweak
 your code and send sample messages for many different test fixtures.
@@ -328,34 +323,30 @@ class `HelloWorldHookTests`:
 
 ```python
 class HelloWorldHookTests(WebhookTestCase):
-    CHANNEL_NAME = "test"
-    URL_TEMPLATE = "/api/v1/external/helloworld?&api_key={api_key}&stream={stream}"
-    DIRECT_MESSAGE_URL_TEMPLATE = "/api/v1/external/helloworld?&api_key={api_key}"
-    WEBHOOK_DIR_NAME = "helloworld"
+    STREAM_NAME = 'test'
+    URL_TEMPLATE = "/api/v1/external/helloworld?&api_key={api_key}"
+    WEBHOOK_DIR_NAME = 'helloworld'
 
     # Note: Include a test function per each distinct message condition your integration supports
     def test_hello_message(self) -> None:
-        expected_topic = "Hello World"
-        expected_message = "Hello! I am happy to be here! :smile:\nThe Wikipedia featured article for today is **[Marilyn Monroe](https://en.wikipedia.org/wiki/Marilyn_Monroe)**"
+        expected_topic = "Hello World";
+        expected_message = "Hello! I am happy to be here! :smile: \nThe Wikipedia featured article for today is **[Marilyn Monroe](https://en.wikipedia.org/wiki/Marilyn_Monroe)**";
 
         # use fixture named helloworld_hello
-        self.check_webhook(
-            "hello",
-            expected_topic,
-            expected_message,
-            content_type="application/x-www-form-urlencoded",
-        )
+        self.check_webhook('hello', expected_topic, expected_message,
+                           content_type="application/x-www-form-urlencoded")
+
 ```
 
-In the above example, `CHANNEL_NAME`, `URL_TEMPLATE`, and `WEBHOOK_DIR_NAME` refer
+In the above example, `STREAM_NAME`, `URL_TEMPLATE`, and `WEBHOOK_DIR_NAME` refer
 to class attributes from the base class, `WebhookTestCase`. These are needed by
 the helper function `check_webhook` to determine how to execute
-your test. `CHANNEL_NAME` should be set to your default channel. If it doesn't exist,
+your test. `STREAM_NAME` should be set to your default stream. If it doesn't exist,
 `check_webhook` will create it while executing your test.
 
-If your test expects a channel name from a test fixture, the value in the fixture
-and the value you set for `CHANNEL_NAME` must match. The test helpers use `CHANNEL_NAME`
-to create the destination channel, and then create the message to send using the
+If your test expects a stream name from a test fixture, the value in the fixture
+and the value you set for `STREAM_NAME` must match. The test helpers use `STREAM_NAME`
+to create the destination stream, and then create the message to send using the
 value from the fixture. If these don't match, the test will fail.
 
 `URL_TEMPLATE` defines how the test runner will call your incoming webhook, in the same way
@@ -372,16 +363,12 @@ class called something like `test_goodbye_message`:
 
 ```python
     def test_goodbye_message(self) -> None:
-        expected_topic = "Hello World"
-        expected_message = "Hello! I am happy to be here! :smile:\nThe Wikipedia featured article for today is **[Goodbye](https://en.wikipedia.org/wiki/Goodbye)**"
+        expected_topic = "Hello World";
+        expected_message = "Hello! I am happy to be here! :smile:\nThe Wikipedia featured article for today is **[Goodbye](https://en.wikipedia.org/wiki/Goodbye)**";
 
         # use fixture named helloworld_goodbye
-        self.check_webhook(
-            "goodbye",
-            expected_topic,
-            expected_message,
-            content_type="application/x-www-form-urlencoded",
-        )
+        self.check_webhook('goodbye', expected_topic, expected_message,
+                           content_type="application/x-www-form-urlencoded")
 ```
 
 As well as a new fixture `goodbye.json` in
@@ -438,13 +425,12 @@ Second, you need to write the actual documentation content in
 ```md
 Learn how Zulip integrations work with this simple Hello World example!
 
-1.  The Hello World webhook will use the `test` channel, which is created
+1.  The Hello World webhook will use the `test` stream, which is created
     by default in the Zulip development environment. If you are running
-    Zulip in production, you should make sure that this channel exists.
+    Zulip in production, you should make sure that this stream exists.
 
-1. {!create-an-incoming-webhook.md!}
+1.  {!create-bot-construct-url.md!}
 
-1. {!generate-integration-url.md!}
 
 1.  To trigger a notification using this example webhook, you can use
     `send_webhook_fixture_message` from a [Zulip development
@@ -469,7 +455,7 @@ Learn how Zulip integrations work with this simple Hello World example!
 
 ```
 
-`{!create-an-incoming-webhook.md!}` and `{!congrats.md!}` are examples of
+`{!create-bot-construct-url.md!}` and `{!congrats.md!}` are examples of
 a Markdown macro. Zulip has a macro-based Markdown/Jinja2 framework that
 includes macros for common instructions in Zulip's webhooks/integrations
 documentation.
@@ -484,24 +470,27 @@ screenshot. Mostly you should plan on templating off an existing guide, like
 
 ## Step 7: Preparing a pull request to zulip/zulip
 
-When you have finished your webhook integration, follow these guidelines before
-pushing the code to your fork and submitting a pull request to zulip/zulip:
+When you have finished your webhook integration and are ready for it to be
+available in the Zulip product, follow these steps to prepare your pull
+request:
 
-- Run tests including linters and ensure you have addressed any issues they
-  report. See [Testing](https://zulip.readthedocs.io/en/latest/testing/testing.html)
-  and [Linters](https://zulip.readthedocs.io/en/latest/testing/linters.html) for details.
-- Read through [Code styles and conventions](
-  https://zulip.readthedocs.io/en/latest/contributing/code-style.html) and take a look
-  through your code to double-check that you've followed Zulip's guidelines.
-- Take a look at your Git history to ensure your commits have been clear and
-  logical (see [Commit discipline](
-  https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html) for tips). If not,
-  consider revising them with `git rebase --interactive`. For most incoming webhooks,
-  you'll want to squash your changes into a single commit and include a good,
-  clear commit message.
+1. Run tests including linters and ensure you have addressed any issues they
+   report. See [Testing](https://zulip.readthedocs.io/en/latest/testing/testing.html)
+   and [Linters](https://zulip.readthedocs.io/en/latest/testing/linters.html) for details.
+2. Read through [Code styles and conventions](
+   https://zulip.readthedocs.io/en/latest/contributing/code-style.html) and take a look
+   through your code to double-check that you've followed Zulip's guidelines.
+3. Take a look at your Git history to ensure your commits have been clear and
+   logical (see [Commit discipline](
+   https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html) for tips). If not,
+   consider revising them with `git rebase --interactive`. For most incoming webhooks,
+   you'll want to squash your changes into a single commit and include a good,
+   clear commit message.
+4. Push code to your fork.
+5. Submit a pull request to zulip/zulip.
 
 If you would like feedback on your integration as you go, feel free to post a
-message on the [public Zulip instance](https://chat.zulip.org/#narrow/channel/integrations).
+message on the [public Zulip instance](https://chat.zulip.org/#narrow/stream/bots).
 You can also create a [draft pull request](
 https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) while you
 are still working on your integration. See the
@@ -531,11 +520,11 @@ def test_unknown_action_no_data(self) -> None:
     # we are testing. The value of result is the error message the webhook should
     # return if no params are sent. The fixture for this test is an empty file.
 
-    # subscribe to the target channel
-    self.subscribe(self.test_user, self.CHANNEL_NAME)
+    # subscribe to the target stream
+    self.subscribe(self.test_user, self.STREAM_NAME)
 
     # post to the webhook url
-    post_params = {'stream_name': self.CHANNEL_NAME,
+    post_params = {'stream_name': self.STREAM_NAME,
                    'content_type': 'application/x-www-form-urlencoded'}
     result = self.client_post(self.url, 'unknown_action', **post_params)
 
@@ -549,8 +538,8 @@ the webhook returns an error, the test fails. Instead, explicitly do the
 setup it would have done, and check the result yourself.
 
 Here, `subscribe_to_stream` is a test helper that uses `TEST_USER_EMAIL` and
-`CHANNEL_NAME` (attributes from the base class) to register the user to receive
-messages in the given channel. If the channel doesn't exist, it creates it.
+`STREAM_NAME` (attributes from the base class) to register the user to receive
+messages in the given stream. If the stream doesn't exist, it creates it.
 
 `client_post`, another helper, performs the HTTP POST that calls the incoming
 webhook. As long as `self.url` is correct, you don't need to construct the webhook
@@ -592,7 +581,7 @@ attribute `TOPIC` as a keyword argument to `build_webhook_url`, like so:
 ```python
 class QuerytestHookTests(WebhookTestCase):
 
-    CHANNEL_NAME = 'querytest'
+    STREAM_NAME = 'querytest'
     TOPIC = "Default topic"
     URL_TEMPLATE = "/api/v1/external/querytest?api_key={api_key}&stream={stream}"
     FIXTURE_DIR_NAME = 'querytest'
@@ -653,8 +642,3 @@ with a string describing the unsupported event type, like so:
 ```
 raise UnsupportedWebhookEventTypeError(event_type)
 ```
-
-## Related articles
-
-* [Integrations overview](/api/integrations-overview)
-* [Incoming webhook integrations](/api/incoming-webhooks-overview)

api_docs/installation-instructions.md

@@ -40,7 +40,3 @@ No download required!
 
 See also [user-contributed client libraries](/api/client-libraries)
 for many other languages.
-
-## Related articles
-
-* [Configuring the Python bindings](/api/configuring-python-bindings)

api_docs/integrations-overview.md

@@ -34,7 +34,7 @@ Zulip.
   [Slack-compatible webhook API](/integrations/slack/slack_incoming).
 
 * If the product can send email notifications, you can
-  [send those emails to a channel](/help/message-a-channel-by-email).
+  [send those emails to a stream](/help/message-a-stream-by-email).
 
 ## Write your own integration
 
@@ -67,11 +67,3 @@ us](/help/contact-support) and we'll see what we can do.
   something that isn't there check out Zulip's
   [REST endpoints](https://github.com/zulip/zulip/blob/main/zproject/urls.py)
   or [contact us](/help/contact-support) and we'll help you out.
-
-## Related articles
-
-* [Bots overview](/help/bots-overview)
-* [Set up integrations](/help/set-up-integrations)
-* [Add a bot or integration](/help/add-a-bot-or-integration)
-* [Generate integration URL](/help/generate-integration-url)
-* [Request an integration](/help/request-an-integration)

api_docs/message-formatting.md

@@ -1,152 +0,0 @@
-# Message formatting
-
-Zulip supports an extended version of Markdown for messages, as well as
-some HTML level special behavior. The Zulip help center article on [message
-formatting](/help/format-your-message-using-markdown) is the primary
-documentation for Zulip's markup features. This article is currently a
-changelog for updates to these features.
-
-The [render a message](/api/render-message) endpoint can be used to get
-the current HTML version of any Markdown syntax for message content.
-
-## Code blocks
-
-**Changes**: As of Zulip 4.0 (feature level 33), [code blocks][help-code]
-can have a `data-code-language` attribute attached to the outer HTML
-`div` element, which records the programming language that was selected
-for syntax highlighting. This field is used in the
-[playgrounds][help-playgrounds] feature for code blocks.
-
-## Global times
-
-**Changes**: In Zulip 3.0 (feature level 8), added [global time
-mentions][help-global-time] to supported Markdown message formatting
-features.
-
-## Image previews
-
-When a Zulip message is sent linking to an uploaded image, Zulip will
-generate an image preview element with the following format.
-
-``` html
-<div class="message_inline_image">
-    <a href="/user_uploads/path/to/image.png" title="image.png">
-        <img data-original-dimensions="1920x1080"
-          src="/user_uploads/thumbnail/path/to/image.png/840x560.webp">
-    </a>
-</div>
-```
-
-If the server has not yet generated thumbnails for the image yet at
-the time the message is sent, the `img` element will be a temporary
-loading indicator image and have the `image-loading-placeholder`
-class, which clients can use to identify loading indicators and
-replace them with a more native loading indicator element if
-desired. For example:
-
-``` html
-<div class="message_inline_image">
-    <a href="/user_uploads/path/to/image.png" title="image.png">
-        <img class="image-loading-placeholder" src="/path/to/spinner.png">
-    </a>
-</div>
-```
-
-Once the server has a working thumbnail, such messages will be updated
-via an `update_message` event, with the `rendering_only: true` flag
-(telling clients not to adjust message edit history), with appropriate
-adjusted `rendered_content`. A client should process those events by
-just using the updated rendering. If thumbnailing failed, the same
-type of event will edit the message's rendered form to remove the
-image preview element, so no special client-side logic should be
-required to process such errors.
-
-Note that in the uncommon situation that the thumbnailing system is
-backlogged, an individual message containing multiple image previews
-may be re-rendered multiple times as each image finishes thumbnailing
-and triggers a message update.
-
-Clients are recommended to do the following when processing image
-previews:
-
-- If the client would like to control the thumbnail resolution used,
-  it can replace the final section of the URL (`840x560.webp` in the
-  example above) with the `name` of its preferred format from the set
-  of supported formats provided by the server in the
-  `server_thumbnail_formats` portion of the `register`
-  response. Clients should not make any assumptions about what format
-  the server will use as the "default" thumbnail resolution, as it may
-  change over time.
-- Download button type elements should provide the original image
-  (encoded via the `href` of the containing `a` tag).
-- Lightbox elements for viewing an image should be designed to
-  immediately display any already-downloaded thumbnail while fetching
-  the original-quality image or an appropriate higher-quality
-  thumbnail from the server, to be transparently swapped in once it is
-  available. Clients that would like to size the lightbox based on the
-  size of the original image can use the `data-original-dimensions`
-  attribute, which encodes the dimensions of the original image as
-  `{width}x{height}`, to do so.  These dimensions are for the image as
-  rendered, _after_ any EXIF rotation and mirroring has been applied.
-- Animated images will have a `data-animated` attribute on the `img`
-  tag. As detailed in `server_thumbnail_formats`, both animated and
-  still images are available for clients to use, depending on their
-  preference. See, for example, the [web
-  setting](/help/allow-image-link-previews#configure-how-animated-images-are-played)
-  to control whether animated images are autoplayed in the message
-  feed.
-- Clients should not assume that the requested format is the format
-  that they will receive; in rare cases where the client has an
-  out-of-date list of `server_thumbnail_formats`, the server will
-  provide an approximation of the client's requested format.  Because
-  of this, clients should not assume that the pixel dimensions or file
-  format match what they requested.
-- No other processing of the URLs is recommended.
-
-**Changes**: In Zulip 9.0 (feature level 276), added
-`data-original-dimensions` attribute to images that have been
-thumbnailed, containing the dimensions of the full-size version of the
-image. Thumbnailing itself was reintroduced at feature level 275.
-
-Previously, with the exception of Zulip servers that used the beta
-Thumbor-based implementation years ago, all image previews in Zulip
-messages were not thumbnailed; the `a` tag and the `img` tag would both
-point to the original image.
-
-Clients that correctly implement the current API should handle
-Thumbor-based older thumbnails correctly, as long as they do not
-assume that `data-original-dimension` is present. Clients should not
-assume that messages sent prior to the introduction of thumbnailing
-have been re-rendered to use the new format or have thumbnails
-available.
-
-## Mentions
-
-**Changes**: In Zulip 9.0 (feature level 247), `channel` was added
-to the supported [wildcard][help-mention-all] options used in the
-[mentions][help-mentions] Markdown message formatting feature.
-
-## Spoilers
-
-**Changes**: In Zulip 3.0 (feature level 15), added
-[spoilers][help-spoilers] to supported Markdown message formatting
-features.
-
-## Removed features
-
-**Changes**: In Zulip 4.0 (feature level 24), the rarely used `!avatar()`
-and `!gravatar()` markup syntax, which was never documented and had an
-inconsistent syntax, were removed.
-
-## Related articles
-
-* [Markdown formatting](/help/format-your-message-using-markdown)
-* [Send a message](/api/send-message)
-* [Render a message](/api/render-message)
-
-[help-code]: /help/code-blocks
-[help-playgrounds]: /help/code-blocks#code-playgrounds
-[help-spoilers]: /help/spoilers
-[help-global-time]: /help/global-times
-[help-mentions]: /help/mention-a-user-or-group
-[help-mention-all]: /help/mention-a-user-or-group#mention-everyone-on-a-channel

api_docs/non-webhook-integrations.md

@@ -4,14 +4,14 @@
 fastest to write, but sometimes a third-party product just doesn't support
 them. Zulip supports several other types of integrations.
 
-* **Python script integrations**
+1. **Python script integrations**
    (examples: SVN, Git), where we can get the service to call our integration
    (by shelling out or otherwise), passing in the required data.  Our preferred
    model for these is to ship these integrations in the
    [Zulip Python API distribution](https://github.com/zulip/python-zulip-api/tree/main/zulip),
    within the `integrations` directory there.
 
-* **Plugin integrations** (examples:
+1. **Plugin integrations** (examples:
    Jenkins, Hubot, Trac) where the user needs to install a plugin into their
    existing software.  These are often more work, but for some products are the
    only way to integrate with the product at all.
@@ -20,7 +20,7 @@ them. Zulip supports several other types of integrations.
     documentation for the third party software in order to learn how to
     write the integration.
 
-* **Interactive bots**. See [Writing bots](/api/writing-bots).
+1. **Interactive bots**. See [Writing bots](/api/writing-bots).
 
 A few notes on how to do these:
 
@@ -51,9 +51,3 @@ examples of ideal UAs are:
 
 * The [general advice](/api/incoming-webhooks-overview#general-advice) for
   webhook integrations applies here as well.
-
-## Related articles
-
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)

api_docs/outgoing-webhooks.md

@@ -2,7 +2,7 @@
 
 Outgoing webhooks allow you to build or set up Zulip integrations
 which are notified when certain types of messages are sent in
-Zulip. When one of those events is triggered, we'll send an HTTP POST
+Zulip. When one of those events is triggered, we'll send a HTTP POST
 payload to the webhook's configured URL.  Webhooks can be used to
 power a wide range of Zulip integrations.  For example, the
 [Zulip Botserver][zulip-botserver] is built on top of this API.
@@ -18,7 +18,7 @@ with porting an existing Slack integration to work with Zulip.
 To register an outgoing webhook:
 
 * Log in to the Zulip server.
-* Navigate to *Personal settings (<i class="zulip-icon zulip-icon-gear"></i>)* -> *Bots* ->
+* Navigate to *Personal settings (<i class="fa fa-cog"></i>)* -> *Bots* ->
   *Add a new bot*.  Select *Outgoing webhook* for bot type, the URL
   you'd like Zulip to post to as the **Endpoint URL**, the format you
   want, and click on *Create bot*. to submit the form/
@@ -29,9 +29,9 @@ To register an outgoing webhook:
 
 There are currently two ways to trigger an outgoing webhook:
 
-*  **@-mention** the bot user in a channel.  If the bot replies, its
-    reply will be sent to that channel and topic.
-*  **Send a direct message** with the bot as one of the recipients.
+1.  **@-mention** the bot user in a stream.  If the bot replies, its
+    reply will be sent to that stream and topic.
+2.  **Send a direct message** with the bot as one of the recipients.
     If the bot replies, its reply will be sent to that thread.
 
 ## Timeouts
@@ -124,11 +124,11 @@ Here's how we fill in the fields that a Slack-format webhook expects:
         </tr>
         <tr>
             <td><code>channel_id</code></td>
-            <td>Channel ID prefixed by "C"</td>
+            <td>Stream ID prefixed by "C"</td>
         </tr>
         <tr>
             <td><code>channel_name</code></td>
-            <td>Channel name</td>
+            <td>Stream name</td>
         </tr>
         <tr>
             <td><code>thread_ts</code></td>

api_docs/real-time-events.md

@@ -4,7 +4,7 @@ Zulip's real-time events API lets you write software that reacts
 immediately to events happening in Zulip.  This API is what powers the
 real-time updates in the Zulip web and mobile apps.  As a result, the
 events available via this API cover all changes to data displayed in
-the Zulip product, from new messages to channel descriptions to
+the Zulip product, from new messages to stream descriptions to
 emoji reactions to changes in user or organization-level settings.
 
 ## Using the events API

api_docs/rest-error-handling.md

@@ -2,48 +2,20 @@
 
 Zulip's API will always return a JSON format response.
 The HTTP status code indicates whether the request was successful
-(200 = success, 4xx = user error, 5xx = server error).
+(200 = success, 40x = user error, 50x = server error).  Every response
+will contain at least two keys: `msg` (a human-readable error message)
+and `result`, which will be either `error` or `success` (this is
+redundant with the HTTP status code, but is convenient when printing
+responses while debugging).
 
-Every response, both success and error responses, will contain at least
-two keys:
+For some common errors, Zulip provides a `code` attribute.  Where
+present, clients should check `code`, rather than `msg`, when looking
+for specific error conditions, since the `msg` strings are
+internationalized (e.g. the server will send the error message
+translated into French if the user has a French locale).
 
-- `msg`: an internationalized, human-readable error message string.
-
-- `result`: either `"error"` or `"success"`, which is redundant with the
-  HTTP status code, but is convenient when print debugging.
-
-Every error response will also contain an additional key:
-
-- `code`: a machine-readable error string, with a default value of
-  `"BAD_REQUEST"` for general errors.
-
-Clients should always check `code`, rather than `msg`, when looking for
-specific error conditions. The string values for `msg` are
-internationalized (e.g., the server will send the error message
-translated into French if the user has a French locale), so checking
-those strings will result in buggy code.
-
-!!! tip ""
-
-     If a client needs information that is only present in the string value
-     of `msg` for a particular error response, then the developers
-     implementing the client should [start a conversation here][api-design]
-     in order to discuss getting a specific error `code` and/or relevant
-     additional key/value pairs for that error response.
-
-In addition to the keys described above, some error responses will
-contain other keys with further details that are useful for clients. The
-specific keys present depend on the error `code`, and are documented at
-the API endpoints where these particular errors appear.
-
-**Changes**: Before Zulip 5.0 (feature level 76), all error responses
-did not contain a `code` key, and its absence indicated that no specific
-error `code` had been allocated for that error.
-
-## Common error responses
-
-Documented below are some error responses that are common to many
-endpoints:
+Each endpoint documents its own unique errors; documented below are
+errors common to many endpoints:
 
 {generate_code_example|/rest-error-handling:post|fixture}
 
@@ -53,12 +25,10 @@ In JSON success responses, all Zulip REST API endpoints may return
 an array of parameters sent in the request that are not supported
 by that specific endpoint.
 
-While this can be expected, e.g., when sending both current and legacy
+While this can be expected, e.g. when sending both current and legacy
 names for a parameter to a Zulip server of unknown version, this often
 indicates either a bug in the client implementation or an attempt to
 configure a new feature while connected to an older Zulip server that
 does not support said feature.
 
 {generate_code_example|/settings:patch|fixture}
-
-[api-design]: https://chat.zulip.org/#narrow/channel/378-api-design

api_docs/rest.md

@@ -6,7 +6,7 @@ you can do in Zulip, you can do with Zulip's REST API.  To use this API:
 * You'll need to [get an API key](/api/api-keys).  You will likely
   want to [create a bot](/help/add-a-bot-or-integration), unless you're
   using the API to interact with
-  your own account (e.g., exporting your personal message history).
+  your own account (e.g. exporting your personal message history).
 * Choose what language you'd like to use.  You can download the
   [Python or JavaScript bindings](/api/installation-instructions), projects in
   [other languages](/api/client-libraries), or

api_docs/roles-and-permissions.md

@@ -76,7 +76,7 @@ event](/api/get-events#realm_user-add), and the
 Many areas of Zulip are customizable by the roles
 above, such as (but not limited to) [restricting message editing and
 deletion](/help/restrict-message-editing-and-deletion) and
-[channels permissions](/help/channel-permissions). The potential
+[streams permissions](/help/stream-permissions). The potential
 permission levels are:
 
 * Everyone / Any user including Guests (least restrictive)
@@ -102,11 +102,6 @@ and owners.
 Note that specific settings and policies in the Zulip API that use these
 permission levels will likely support a subset of those listed above.
 
-## Group-based permissions
-
-Some settings have been migrated to a more flexible system based on
-[user groups](/api/group-setting-values).
-
 ## Determining if a user is a full member
 
 When a Zulip organization has set up a [waiting period before new members

api_docs/running-bots.md

@@ -12,7 +12,7 @@ https://github.com/zulip/python-zulip-api/tree/main/zulip_bots/zulip_bots/bots).
 You'll need:
 
 * An account in a Zulip organization
-  (e.g., [the Zulip development community](https://zulip.com/development-community/),
+  (e.g. [the Zulip development community](https://zulip.com/development-community/),
   `<yourSubdomain>.zulipchat.com`, or a Zulip organization on your own
   [development](https://zulip.readthedocs.io/en/latest/development/overview.html) or
   [production](https://zulip.readthedocs.io/en/latest/production/install.html) server).
@@ -20,42 +20,39 @@ You'll need:
 
 **Note: Please be considerate when testing experimental bots on public servers such as chat.zulip.org.**
 
-{start_tabs}
+1. Go to your Zulip account and
+   [add a bot](/help/add-a-bot-or-integration). Use **Generic bot** as the bot type.
 
-1. [Create a bot](/help/add-a-bot-or-integration), making sure to select
-   **Generic bot** as the **Bot type**.
+1. Download the bot's `zuliprc` configuration file to your computer.
 
-1. [Download the bot's `zuliprc` file](/api/configuring-python-bindings#download-a-zuliprc-file).
+1. Download the `zulip_bots` Python package to your computer using `pip3 install zulip_bots`.
 
-1. Use the following command to install the
-   [`zulip_bots` Python package](https://pypi.org/project/zulip-bots/):
+     *Note: Click
+     [here](
+     writing-bots#installing-a-development-version-of-the-zulip-bots-package)
+     to install the latest development version of the package.*
 
-        pip3 install zulip_bots
+1. Start the bot process on your computer.
 
-1. Use the following command to start the bot process *(replacing
-   `~/path/to/zuliprc` with the path to the `zuliprc` file you downloaded above)*:
+    * Run
+      ```
+      zulip-run-bot <bot-name> --config-file ~/path/to/zuliprc
+      ```
 
-        zulip-run-bot <bot-name> --config-file ~/path/to/zuliprc
+      (replacing `~/path/to/zuliprc` with the path to the `zuliprc` file you downloaded above).
 
-1. Check the output of the command above to make sure your bot is running.
-   It should include the following line:
+    * Check the output of the command. It should include the following line:
 
-        INFO:root:starting message handling...
+            INFO:root:starting message handling...
 
-1. Test your setup by [starting a new direct message](/help/starting-a-new-direct-message)
-   with the bot or [mentioning](/help/mention-a-user-or-group) the bot on a channel.
+        Congrats! Your bot is running.
 
-!!! tip ""
-
-    To use the latest development version of the `zulip_bots` package, follow
-    [these steps](writing-bots#installing-a-development-version-of-the-zulip-bots-package).
-
-{end_tabs}
+1. To talk with the bot, at-mention its name, like `@**bot-name**`.
 
 You can now play around with the bot and get it configured the way you
 like.  Eventually, you'll probably want to run it in a production
-environment where it'll stay up, by [deploying](/api/deploying-bots) it on a
-server using the Zulip Botserver.
+environment where it'll stay up, by [deploying](/api/deploying-bots) it on a server using the
+Zulip Botserver.
 
 ## Common problems
 
@@ -66,9 +63,3 @@ server using the Zulip Botserver.
       the Vagrant environment.
     * Some bots require Python 3. Try switching to a Python 3 environment before running
       your bot.
-
-## Related articles
-
-* [Non-webhook integrations](/api/non-webhook-integrations)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)

api_docs/send-message.md

@@ -11,7 +11,7 @@
 {tab|curl}
 
 ``` curl
-# For channel messages
+# For stream messages
 curl -X POST {{ api_url }}/v1/messages \
     -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
     --data-urlencode type=stream \
@@ -34,7 +34,7 @@ You can use `zulip-send`
 the command-line, providing the message content via STDIN.
 
 ```bash
-# For channel messages
+# For stream messages
 zulip-send --stream Denmark --subject Castle \
     --user othello-bot@example.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
 

api_docs/sidebar_index.md

@@ -21,8 +21,6 @@
 * [HTTP headers](/api/http-headers)
 * [Error handling](/api/rest-error-handling)
 * [Roles and permissions](/api/roles-and-permissions)
-* [Group-setting values](/api/group-setting-values)
-* [Message formatting](/api/message-formatting)
 * [Client libraries](/api/client-libraries)
 * [API changelog](/api/changelog)
 

api_docs/writing-bots.md

@@ -17,41 +17,23 @@ On this page you'll find:
 
 ## Installing a development version of the Zulip bots package
 
-{start_tabs}
+1. `git clone https://github.com/zulip/python-zulip-api.git` - clone the [python-zulip-api](
+  https://github.com/zulip/python-zulip-api) repository.
 
-1. Clone the [python-zulip-api](https://github.com/zulip/python-zulip-api)
-   repository:
+2. `cd python-zulip-api` - navigate into your cloned repository.
 
-    ```
-    git clone https://github.com/zulip/python-zulip-api.git
-    ```
+3. `python3 ./tools/provision` - install all requirements in a Python virtualenv.
 
-1. Navigate into your cloned repository:
+4. The output of `provision` will end with a command of the form `source .../activate`;
+   run that command to enter the new virtualenv.
 
-    ```
-    cd python-zulip-api
-    ```
+5. *Finished*. You should now see the name of your venv preceding your prompt,
+   e.g. `(zulip-api-py3-venv)`.
 
-1. Install all requirements in a Python virtualenv:
-
-    ```
-    python3 ./tools/provision
-    ```
-
-1. Run the command provided in the final output of the `provision` process to
-   enter the new virtualenv. The command will be of the form `source .../activate`.
-
-1. You should now see the name of your virtualenv preceding your prompt (e.g.,
-   `(zulip-api-py3-venv)`).
-
-!!! tip ""
-
-    `provision` installs the `zulip`, `zulip_bots`, and
-    `zulip_botserver` packages in developer mode. This enables you to
-    modify these packages and then run your modified code without
-    having to first re-install the packages or re-provision.
-
-{end_tabs}
+*Hint: `provision` installs the `zulip`, `zulip_bots`, and
+ `zulip_botserver` packages in developer mode. This enables you to
+ modify these packages and then run your modified code without
+ having to first re-install the packages or re-provision.*
 
 ## Writing a bot
 
@@ -151,7 +133,7 @@ Response: stream: followup topic: foo_sender@zulip.com
 ```
 
 Note that the `-b` (aka `--bot-config-file`) argument is for an optional third party
-config file (e.g., ~/giphy.conf), which only applies to certain types of bots.
+config file (e.g. ~/giphy.conf), which only applies to certain types of bots.
 
 ## Bot API
 
@@ -191,7 +173,7 @@ def usage(self):
         This plugin will allow users to flag messages
         as being follow-up items.  Users should preface
         messages with "@followup".
-        Before running this, make sure to create a channel
+        Before running this, make sure to create a stream
         called "followup" that your API user can send to.
         '''
 ```
@@ -208,7 +190,7 @@ handles user message.
 
 * message - a dictionary describing a Zulip message
 
-* bot_handler - used to interact with the server, e.g., to send a message
+* bot_handler - used to interact with the server, e.g. to send a message
 
 #### Return values
 
@@ -247,7 +229,7 @@ about where the message is sent to.
 ```python
 bot_handler.send_message(dict(
     type='stream', # can be 'stream' or 'private'
-    to=channel_name, # either the channel name or user's email
+    to=stream_name, # either the stream name or user's email
     subject=subject, # message subject
     content=message, # content of the sent message
 ))
@@ -290,7 +272,7 @@ bot_handler.update_message(dict(
 ### bot_handler.storage
 
 A common problem when writing an interactive bot is that you want to
-be able to store a bit of persistent state for the bot (e.g., for an
+be able to store a bit of persistent state for the bot (e.g. for an
 RSVP bot, the RSVPs).  For a sufficiently complex bot, you want need
 your own database, but for simpler bots, we offer a convenient way for
 bot code to persistently store data.
@@ -382,7 +364,7 @@ every call to `put` and `get`, respectively.
 * key - the API key you created for the bot; this is how Zulip knows
   the request is from an authorized user.
 
-* email - the email address of the bot, e.g., `some-bot@zulip.com`
+* email - the email address of the bot, e.g. `some-bot@zulip.com`
 
 * site - your development environment URL; if you are working on a
   development environment hosted on your computer, use
@@ -530,10 +512,3 @@ The long-term plan for this bot system is to allow the same
   received from Zulip's outgoing webhooks integration.
 * For bots merged into the mainline Zulip codebase, enabled via a
   button in the Zulip web UI, with no code deployment effort required.
-
-## Related articles
-
-* [Non-webhook integrations](/api/non-webhook-integrations)
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Configuring the Python bindings](/api/configuring-python-bindings)

confirmation/migrations/0012_alter_confirmation_id.py

@@ -1,17 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("confirmation", "0011_alter_confirmation_expiry_date"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="confirmation",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

confirmation/migrations/0013_alter_realmcreationkey_id.py

@@ -1,17 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("confirmation", "0012_alter_confirmation_id"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="realmcreationkey",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

confirmation/models.py

@@ -1,11 +1,10 @@
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
 __revision__ = "$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $"
+import datetime
 import secrets
 from base64 import b32encode
-from collections.abc import Mapping
-from datetime import timedelta
-from typing import Optional, TypeAlias, Union, cast
+from typing import List, Mapping, Optional, Union
 from urllib.parse import urljoin
 
 from django.conf import settings
@@ -17,7 +16,7 @@ from django.http import HttpRequest, HttpResponse
 from django.template.response import TemplateResponse
 from django.urls import reverse
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
+from typing_extensions import TypeAlias, override
 
 from confirmation import settings as confirmation_settings
 from zerver.lib.types import UnspecifiedValue
@@ -31,12 +30,6 @@ from zerver.models import (
     UserProfile,
 )
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import (
-        PreregistrationRemoteRealmBillingUser,
-        PreregistrationRemoteServerBillingUser,
-    )
-
 
 class ConfirmationKeyError(Exception):
     WRONG_LENGTH = 1
@@ -63,25 +56,18 @@ def generate_key() -> str:
     return b32encode(secrets.token_bytes(15)).decode().lower()
 
 
-NoZilencerConfirmationObjT: TypeAlias = (
-    MultiuseInvite
-    | PreregistrationRealm
-    | PreregistrationUser
-    | EmailChangeStatus
-    | UserProfile
-    | RealmReactivationStatus
-)
-ZilencerConfirmationObjT: TypeAlias = Union[
-    NoZilencerConfirmationObjT,
-    "PreregistrationRemoteServerBillingUser",
-    "PreregistrationRemoteRealmBillingUser",
+ConfirmationObjT: TypeAlias = Union[
+    MultiuseInvite,
+    PreregistrationRealm,
+    PreregistrationUser,
+    EmailChangeStatus,
+    UserProfile,
+    RealmReactivationStatus,
 ]
 
-ConfirmationObjT: TypeAlias = NoZilencerConfirmationObjT | ZilencerConfirmationObjT
-
 
 def get_object_from_key(
-    confirmation_key: str, confirmation_types: list[int], *, mark_object_used: bool
+    confirmation_key: str, confirmation_types: List[int], *, mark_object_used: bool
 ) -> ConfirmationObjT:
     """Access a confirmation object from one of the provided confirmation
     types with the provided key.
@@ -125,25 +111,21 @@ def get_object_from_key(
     return obj
 
 
-def create_confirmation_object(
+def create_confirmation_link(
     obj: ConfirmationObjT,
     confirmation_type: int,
     *,
-    validity_in_minutes: int | None | UnspecifiedValue = UnspecifiedValue(),
-    no_associated_realm_object: bool = False,
-) -> "Confirmation":
+    validity_in_minutes: Union[Optional[int], UnspecifiedValue] = UnspecifiedValue(),
+    url_args: Mapping[str, str] = {},
+    realm_creation: bool = False,
+) -> str:
     # validity_in_minutes is an override for the default values which are
     # determined by the confirmation_type - its main purpose is for use
     # in tests which may want to have control over the exact expiration time.
     key = generate_key()
-
-    # Some confirmation objects, like those for realm creation or those used
-    # for the self-hosted management flows, are not associated with a realm
-    # hosted by this Zulip server.
-    if no_associated_realm_object:
+    if realm_creation:
         realm = None
     else:
-        obj = cast(NoZilencerConfirmationObjT, obj)
         assert not isinstance(obj, PreregistrationRealm)
         realm = obj.realm
 
@@ -154,11 +136,13 @@ def create_confirmation_object(
             expiry_date = None
         else:
             assert validity_in_minutes is not None
-            expiry_date = current_time + timedelta(minutes=validity_in_minutes)
+            expiry_date = current_time + datetime.timedelta(minutes=validity_in_minutes)
     else:
-        expiry_date = current_time + timedelta(days=_properties[confirmation_type].validity_in_days)
+        expiry_date = current_time + datetime.timedelta(
+            days=_properties[confirmation_type].validity_in_days
+        )
 
-    return Confirmation.objects.create(
+    Confirmation.objects.create(
         content_object=obj,
         date_sent=current_time,
         confirmation_key=key,
@@ -166,43 +150,19 @@ def create_confirmation_object(
         expiry_date=expiry_date,
         type=confirmation_type,
     )
-
-
-def create_confirmation_link(
-    obj: ConfirmationObjT,
-    confirmation_type: int,
-    *,
-    validity_in_minutes: int | None | UnspecifiedValue = UnspecifiedValue(),
-    url_args: Mapping[str, str] = {},
-    no_associated_realm_object: bool = False,
-) -> str:
-    return confirmation_url_for(
-        create_confirmation_object(
-            obj,
-            confirmation_type,
-            validity_in_minutes=validity_in_minutes,
-            no_associated_realm_object=no_associated_realm_object,
-        ),
-        url_args=url_args,
-    )
-
-
-def confirmation_url_for(confirmation_obj: "Confirmation", url_args: Mapping[str, str] = {}) -> str:
-    return confirmation_url(
-        confirmation_obj.confirmation_key, confirmation_obj.realm, confirmation_obj.type, url_args
-    )
+    return confirmation_url(key, realm, confirmation_type, url_args)
 
 
 def confirmation_url(
     confirmation_key: str,
-    realm: Realm | None,
+    realm: Optional[Realm],
     confirmation_type: int,
     url_args: Mapping[str, str] = {},
 ) -> str:
     url_args = dict(url_args)
     url_args["confirmation_key"] = confirmation_key
     return urljoin(
-        settings.ROOT_DOMAIN_URI if realm is None else realm.url,
+        settings.ROOT_DOMAIN_URI if realm is None else realm.uri,
         reverse(_properties[confirmation_type].url_name, kwargs=url_args),
     )
 
@@ -225,8 +185,6 @@ class Confirmation(models.Model):
     MULTIUSE_INVITE = 6
     REALM_CREATION = 7
     REALM_REACTIVATION = 8
-    REMOTE_SERVER_BILLING_LEGACY_LOGIN = 9
-    REMOTE_REALM_BILLING_LEGACY_LOGIN = 10
     type = models.PositiveSmallIntegerField()
 
     class Meta:
@@ -263,13 +221,6 @@ _properties = {
     Confirmation.REALM_CREATION: ConfirmationType("get_prereg_key_and_redirect"),
     Confirmation.REALM_REACTIVATION: ConfirmationType("realm_reactivation"),
 }
-if settings.ZILENCER_ENABLED:
-    _properties[Confirmation.REMOTE_SERVER_BILLING_LEGACY_LOGIN] = ConfirmationType(
-        "remote_billing_legacy_server_from_login_confirmation_link"
-    )
-    _properties[Confirmation.REMOTE_REALM_BILLING_LEGACY_LOGIN] = ConfirmationType(
-        "remote_realm_billing_from_login_confirmation_link"
-    )
 
 
 def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> str:
@@ -291,7 +242,7 @@ def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> st
 # add another Confirmation.type for this; it's this way for historical reasons.
 
 
-def validate_key(creation_key: str | None) -> Optional["RealmCreationKey"]:
+def validate_key(creation_key: Optional[str]) -> Optional["RealmCreationKey"]:
     """Get the record for this key, raising InvalidCreationKey if non-None but invalid."""
     if creation_key is None:
         return None

corporate/lib/activity.py

@@ -1,402 +0,0 @@
-from collections import defaultdict
-from collections.abc import Callable, Sequence
-from dataclasses import dataclass
-from datetime import datetime
-from decimal import Decimal
-from typing import Any
-from urllib.parse import urlencode
-
-from django.conf import settings
-from django.db import connection
-from django.db.backends.utils import CursorWrapper
-from django.db.models import Prefetch
-from django.template import loader
-from django.urls import reverse
-from django.utils.timezone import now as timezone_now
-from markupsafe import Markup
-from psycopg2.sql import Composable
-
-from corporate.lib.stripe import (
-    RealmBillingSession,
-    RemoteRealmBillingSession,
-    RemoteServerBillingSession,
-)
-from corporate.models import CustomerPlan, LicenseLedger
-from zerver.lib.pysa import mark_sanitized
-from zerver.lib.url_encoding import append_url_query_string
-from zerver.models import Realm
-from zilencer.models import (
-    RemoteCustomerUserCount,
-    RemoteRealm,
-    RemoteRealmAuditLog,
-    RemoteZulipServer,
-    get_remote_customer_user_count,
-)
-
-
-@dataclass
-class RemoteActivityPlanData:
-    current_status: str
-    current_plan_name: str
-    annual_revenue: int
-    rate: str
-
-
-def make_table(
-    title: str,
-    cols: Sequence[str],
-    rows: Sequence[Any],
-    *,
-    totals: Any | None = None,
-    stats_link: Markup | None = None,
-    has_row_class: bool = False,
-) -> str:
-    if not has_row_class:
-
-        def fix_row(row: Any) -> dict[str, Any]:
-            return dict(cells=row, row_class=None)
-
-        rows = list(map(fix_row, rows))
-
-    data = dict(title=title, cols=cols, rows=rows, totals=totals, stats_link=stats_link)
-
-    content = loader.render_to_string(
-        "corporate/activity/activity_table.html",
-        dict(data=data),
-    )
-
-    return content
-
-
-def fix_rows(
-    rows: list[list[Any]],
-    i: int,
-    fixup_func: Callable[[str], Markup] | Callable[[datetime], str] | Callable[[int], int],
-) -> None:
-    for row in rows:
-        row[i] = fixup_func(row[i])
-
-
-def get_query_data(query: Composable) -> list[list[Any]]:
-    cursor = connection.cursor()
-    cursor.execute(query)
-    rows = cursor.fetchall()
-    rows = list(map(list, rows))
-    cursor.close()
-    return rows
-
-
-def dictfetchall(cursor: CursorWrapper) -> list[dict[str, Any]]:
-    """Returns all rows from a cursor as a dict"""
-    desc = cursor.description
-    return [dict(zip((col[0] for col in desc), row, strict=False)) for row in cursor.fetchall()]
-
-
-def format_optional_datetime(date: datetime | None, display_none: bool = False) -> str:
-    if date:
-        return date.strftime("%Y-%m-%d %H:%M")
-    elif display_none:
-        return "None"
-    else:
-        return ""
-
-
-def format_datetime_as_date(date: datetime) -> str:
-    return date.strftime("%Y-%m-%d")
-
-
-def format_none_as_zero(value: int | None) -> int:
-    if value:
-        return value
-    else:
-        return 0
-
-
-def user_activity_link(email: str, user_profile_id: int) -> Markup:
-    from corporate.views.user_activity import get_user_activity
-
-    url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
-    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
-
-
-def realm_activity_link(realm_str: str) -> Markup:
-    from corporate.views.realm_activity import get_realm_activity
-
-    url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
-
-
-def realm_stats_link(realm_str: str) -> Markup:
-    from analytics.views.stats import stats_for_realm
-
-    url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
-
-
-def realm_support_link(realm_str: str) -> Markup:
-    support_url = reverse("support")
-    query = urlencode({"q": realm_str})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}"><i class="fa fa-gear"></i></a>').format(url=url)
-
-
-def realm_url_link(realm_str: str) -> Markup:
-    host = Realm.host_for_subdomain(realm_str)
-    url = settings.EXTERNAL_URI_SCHEME + mark_sanitized(host)
-    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
-
-
-def remote_installation_stats_link(server_id: int) -> Markup:
-    from analytics.views.stats import stats_for_remote_installation
-
-    url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
-
-
-def remote_installation_support_link(hostname: str) -> Markup:
-    support_url = reverse("remote_servers_support")
-    query = urlencode({"q": hostname})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}"><i class="fa fa-gear"></i></a>').format(url=url)
-
-
-def get_plan_rate_percentage(discount: str | None) -> str:
-    # CustomerPlan.discount is a string field that stores the discount.
-    if discount is None or discount == "0":
-        return "100%"
-
-    rate = 100 - Decimal(discount)
-    if rate * 100 % 100 == 0:
-        precision = 0
-    else:
-        precision = 2
-    return f"{rate:.{precision}f}%"
-
-
-def get_remote_activity_plan_data(
-    plan: CustomerPlan,
-    license_ledger: LicenseLedger,
-    *,
-    remote_realm: RemoteRealm | None = None,
-    remote_server: RemoteZulipServer | None = None,
-) -> RemoteActivityPlanData:
-    if plan.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY or plan.status in (
-        CustomerPlan.DOWNGRADE_AT_END_OF_FREE_TRIAL,
-        CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE,
-    ):
-        renewal_cents = 0
-        current_rate = "---"
-    elif plan.tier == CustomerPlan.TIER_SELF_HOSTED_COMMUNITY:
-        renewal_cents = 0
-        current_rate = "0%"
-    elif remote_realm is not None:
-        renewal_cents = RemoteRealmBillingSession(
-            remote_realm=remote_realm
-        ).get_annual_recurring_revenue_for_support_data(plan, license_ledger)
-        current_rate = get_plan_rate_percentage(plan.discount)
-    else:
-        assert remote_server is not None
-        renewal_cents = RemoteServerBillingSession(
-            remote_server=remote_server
-        ).get_annual_recurring_revenue_for_support_data(plan, license_ledger)
-        current_rate = get_plan_rate_percentage(plan.discount)
-
-    return RemoteActivityPlanData(
-        current_status=plan.get_plan_status_as_text(),
-        current_plan_name=plan.name,
-        annual_revenue=renewal_cents,
-        rate=current_rate,
-    )
-
-
-def get_estimated_arr_and_rate_by_realm() -> tuple[dict[str, int], dict[str, str]]:  # nocoverage
-    # NOTE: Customers without a plan might still have a discount attached to them which
-    # are not included in `plan_rate`.
-    annual_revenue = {}
-    plan_rate = {}
-    plans = (
-        CustomerPlan.objects.filter(
-            status=CustomerPlan.ACTIVE,
-            customer__remote_realm__isnull=True,
-            customer__remote_server__isnull=True,
-        )
-        .prefetch_related(
-            Prefetch(
-                "licenseledger_set",
-                queryset=LicenseLedger.objects.order_by("plan", "-id").distinct("plan"),
-                to_attr="latest_ledger_entry",
-            )
-        )
-        .select_related("customer__realm")
-    )
-
-    for plan in plans:
-        assert plan.customer.realm is not None
-        latest_ledger_entry = plan.latest_ledger_entry[0]  # type: ignore[attr-defined] # attribute from prefetch_related query
-        assert latest_ledger_entry is not None
-        renewal_cents = RealmBillingSession(
-            realm=plan.customer.realm
-        ).get_annual_recurring_revenue_for_support_data(plan, latest_ledger_entry)
-        annual_revenue[plan.customer.realm.string_id] = renewal_cents
-        plan_rate[plan.customer.realm.string_id] = get_plan_rate_percentage(plan.discount)
-    return annual_revenue, plan_rate
-
-
-def get_plan_data_by_remote_server() -> dict[int, RemoteActivityPlanData]:  # nocoverage
-    remote_server_plan_data: dict[int, RemoteActivityPlanData] = {}
-    plans = (
-        CustomerPlan.objects.filter(
-            status__lt=CustomerPlan.LIVE_STATUS_THRESHOLD,
-            customer__realm__isnull=True,
-            customer__remote_realm__isnull=True,
-            customer__remote_server__deactivated=False,
-        )
-        .prefetch_related(
-            Prefetch(
-                "licenseledger_set",
-                queryset=LicenseLedger.objects.order_by("plan", "-id").distinct("plan"),
-                to_attr="latest_ledger_entry",
-            )
-        )
-        .select_related("customer__remote_server")
-    )
-
-    for plan in plans:
-        server_id = None
-        assert plan.customer.remote_server is not None
-        server_id = plan.customer.remote_server.id
-        assert server_id is not None
-
-        latest_ledger_entry = plan.latest_ledger_entry[0]  # type: ignore[attr-defined] # attribute from prefetch_related query
-        assert latest_ledger_entry is not None
-
-        plan_data = get_remote_activity_plan_data(
-            plan, latest_ledger_entry, remote_server=plan.customer.remote_server
-        )
-
-        current_data = remote_server_plan_data.get(server_id)
-        if current_data is not None:
-            current_revenue = remote_server_plan_data[server_id].annual_revenue
-            current_plans = remote_server_plan_data[server_id].current_plan_name
-            # There should only ever be one CustomerPlan for a remote server with
-            # a status that is less than the CustomerPlan.LIVE_STATUS_THRESHOLD.
-            remote_server_plan_data[server_id] = RemoteActivityPlanData(
-                current_status="ERROR: MULTIPLE PLANS",
-                current_plan_name=f"{current_plans}, {plan_data.current_plan_name}",
-                annual_revenue=current_revenue + plan_data.annual_revenue,
-                rate="",
-            )
-        else:
-            remote_server_plan_data[server_id] = plan_data
-    return remote_server_plan_data
-
-
-def get_plan_data_by_remote_realm() -> dict[int, dict[int, RemoteActivityPlanData]]:  # nocoverage
-    remote_server_plan_data_by_realm: dict[int, dict[int, RemoteActivityPlanData]] = {}
-    plans = (
-        CustomerPlan.objects.filter(
-            status__lt=CustomerPlan.LIVE_STATUS_THRESHOLD,
-            customer__realm__isnull=True,
-            customer__remote_server__isnull=True,
-            customer__remote_realm__is_system_bot_realm=False,
-            customer__remote_realm__realm_deactivated=False,
-        )
-        .prefetch_related(
-            Prefetch(
-                "licenseledger_set",
-                queryset=LicenseLedger.objects.order_by("plan", "-id").distinct("plan"),
-                to_attr="latest_ledger_entry",
-            )
-        )
-        .select_related("customer__remote_realm")
-    )
-
-    for plan in plans:
-        server_id = None
-        assert plan.customer.remote_realm is not None
-        server_id = plan.customer.remote_realm.server_id
-        assert server_id is not None
-
-        latest_ledger_entry = plan.latest_ledger_entry[0]  # type: ignore[attr-defined] # attribute from prefetch_related query
-        assert latest_ledger_entry is not None
-
-        plan_data = get_remote_activity_plan_data(
-            plan, latest_ledger_entry, remote_realm=plan.customer.remote_realm
-        )
-
-        current_server_data = remote_server_plan_data_by_realm.get(server_id)
-        realm_id = plan.customer.remote_realm.id
-
-        if current_server_data is None:
-            realm_dict = {realm_id: plan_data}
-            remote_server_plan_data_by_realm[server_id] = realm_dict
-        else:
-            assert current_server_data is not None
-            current_realm_data = current_server_data.get(realm_id)
-            if current_realm_data is not None:
-                # There should only ever be one CustomerPlan for a remote realm with
-                # a status that is less than the CustomerPlan.LIVE_STATUS_THRESHOLD.
-                current_revenue = current_realm_data.annual_revenue
-                current_plans = current_realm_data.current_plan_name
-                current_server_data[realm_id] = RemoteActivityPlanData(
-                    current_status="ERROR: MULTIPLE PLANS",
-                    current_plan_name=f"{current_plans}, {plan_data.current_plan_name}",
-                    annual_revenue=current_revenue + plan_data.annual_revenue,
-                    rate="",
-                )
-            else:
-                current_server_data[realm_id] = plan_data
-
-    return remote_server_plan_data_by_realm
-
-
-def get_remote_realm_user_counts(
-    event_time: datetime = timezone_now(),
-) -> dict[int, RemoteCustomerUserCount]:  # nocoverage
-    user_counts_by_realm: dict[int, RemoteCustomerUserCount] = {}
-    for log in (
-        RemoteRealmAuditLog.objects.filter(
-            event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
-            event_time__lte=event_time,
-            remote_realm__isnull=False,
-        )
-        # Important: extra_data is empty for some pre-2020 audit logs
-        # prior to the introduction of realm_user_count_by_role
-        # logging. Meanwhile, modern Zulip servers using
-        # bulk_create_users to create the users in the system bot
-        # realm also generate such audit logs. Such audit logs should
-        # never be the latest in a normal realm.
-        .exclude(extra_data={})
-        .order_by("remote_realm", "-event_time")
-        .distinct("remote_realm")
-        .select_related("remote_realm")
-    ):
-        assert log.remote_realm is not None
-        user_counts_by_realm[log.remote_realm.id] = get_remote_customer_user_count([log])
-
-    return user_counts_by_realm
-
-
-def get_remote_server_audit_logs(
-    event_time: datetime = timezone_now(),
-) -> dict[int, list[RemoteRealmAuditLog]]:
-    logs_per_server: dict[int, list[RemoteRealmAuditLog]] = defaultdict(list)
-    for log in (
-        RemoteRealmAuditLog.objects.filter(
-            event_type__in=RemoteRealmAuditLog.SYNCED_BILLING_EVENTS,
-            event_time__lte=event_time,
-        )
-        # Important: extra_data is empty for some pre-2020 audit logs
-        # prior to the introduction of realm_user_count_by_role
-        # logging. Meanwhile, modern Zulip servers using
-        # bulk_create_users to create the users in the system bot
-        # realm also generate such audit logs. Such audit logs should
-        # never be the latest in a normal realm.
-        .exclude(extra_data={})
-        .order_by("server_id", "realm_id", "-event_time")
-        .distinct("server_id", "realm_id")
-        .select_related("server")
-    ):
-        logs_per_server[log.server.id].append(log)
-
-    return logs_per_server

corporate/lib/decorator.py

@@ -1,204 +0,0 @@
-from collections.abc import Callable
-from functools import wraps
-from typing import Concatenate
-from urllib.parse import urlencode, urljoin
-
-from django.conf import settings
-from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse
-from django.shortcuts import render
-from django.urls import reverse
-from typing_extensions import ParamSpec
-
-from corporate.lib.remote_billing_util import (
-    RemoteBillingIdentityExpiredError,
-    get_remote_realm_and_user_from_session,
-    get_remote_server_and_user_from_session,
-)
-from corporate.lib.stripe import RemoteRealmBillingSession, RemoteServerBillingSession
-from zerver.lib.exceptions import RemoteBillingAuthenticationError
-from zerver.lib.subdomains import get_subdomain
-from zerver.lib.url_encoding import append_url_query_string
-from zilencer.models import RemoteRealm
-
-ParamT = ParamSpec("ParamT")
-
-
-def session_expired_ajax_response(login_url: str) -> JsonResponse:  # nocoverage
-    return JsonResponse(
-        {
-            "error_message": "Remote billing authentication expired",
-            "login_url": login_url,
-        },
-        status=401,
-    )
-
-
-def is_self_hosting_management_subdomain(request: HttpRequest) -> bool:
-    subdomain = get_subdomain(request)
-    return subdomain == settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN
-
-
-def self_hosting_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, ParamT], HttpResponse],
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest, /, *args: ParamT.args, **kwargs: ParamT.kwargs
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-        return view_func(request, *args, **kwargs)
-
-    return _wrapped_view_func
-
-
-def authenticated_remote_realm_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, RemoteRealmBillingSession, ParamT], HttpResponse],
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest,
-        /,
-        *args: ParamT.args,
-        **kwargs: ParamT.kwargs,
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-
-        realm_uuid = kwargs.get("realm_uuid")
-        if realm_uuid is not None and not isinstance(realm_uuid, str):  # nocoverage
-            raise TypeError("realm_uuid must be a string or None")
-
-        try:
-            remote_realm, remote_billing_user = get_remote_realm_and_user_from_session(
-                request, realm_uuid
-            )
-        except RemoteBillingIdentityExpiredError as e:
-            # The user had an authenticated session with an identity_dict,
-            # but it expired.
-            # We want to redirect back to the start of their login flow
-            # at their {realm.host}/self-hosted-billing/ with a proper
-            # next parameter to take them back to what they're trying
-            # to access after re-authing.
-            # Note: Theoretically we could take the realm_uuid from the request
-            # path or params to figure out the remote_realm.host for the redirect,
-            # but that would mean leaking that .host value to anyone who knows
-            # the uuid. Therefore we limit ourselves to taking the realm_uuid
-            # from the identity_dict - since that proves that the user at least
-            # previously was successfully authenticated as a billing admin of that
-            # realm.
-            realm_uuid = e.realm_uuid
-            server_uuid = e.server_uuid
-            uri_scheme = e.uri_scheme
-            if realm_uuid is None:
-                # This doesn't make sense - if get_remote_realm_and_user_from_session
-                # found an expired identity dict, it should have had a realm_uuid.
-                raise AssertionError
-
-            assert server_uuid is not None, "identity_dict with realm_uuid must have server_uuid"
-            assert uri_scheme is not None, "identity_dict with realm_uuid must have uri_scheme"
-
-            try:
-                remote_realm = RemoteRealm.objects.get(uuid=realm_uuid, server__uuid=server_uuid)
-            except RemoteRealm.DoesNotExist:
-                # This should be impossible - unless the RemoteRealm existed and somehow the row
-                # was deleted.
-                raise AssertionError
-
-            # Using EXTERNAL_URI_SCHEME means we'll do https:// in production, which is
-            # the sane default - while having http:// in development, which will allow
-            # these redirects to work there for testing.
-            url = urljoin(uri_scheme + remote_realm.host, "/self-hosted-billing/")
-
-            page_type = get_next_page_param_from_request_path(request)
-            if page_type is not None:
-                query = urlencode({"next_page": page_type})
-                url = append_url_query_string(url, query)
-
-            # Return error for AJAX requests with url.
-            if request.headers.get("x-requested-with") == "XMLHttpRequest":  # nocoverage
-                return session_expired_ajax_response(url)
-
-            return HttpResponseRedirect(url)
-
-        billing_session = RemoteRealmBillingSession(
-            remote_realm, remote_billing_user=remote_billing_user
-        )
-        return view_func(request, billing_session)
-
-    return _wrapped_view_func
-
-
-def get_next_page_param_from_request_path(request: HttpRequest) -> str | None:
-    # Our endpoint URLs in this subsystem end with something like
-    # /sponsorship or /plans etc.
-    # Therefore we can use this nice property to figure out easily what
-    # kind of page the user is trying to access and find the right value
-    # for the `next` query parameter.
-    path = request.path
-    if path.endswith("/"):
-        path = path[:-1]
-
-    page_type = path.split("/")[-1]
-
-    from corporate.views.remote_billing_page import (
-        VALID_NEXT_PAGES as REMOTE_BILLING_VALID_NEXT_PAGES,
-    )
-
-    if page_type in REMOTE_BILLING_VALID_NEXT_PAGES:
-        return page_type
-
-    # page_type is not where we want user to go after a login, so just render the default page.
-    return None  # nocoverage
-
-
-def authenticated_remote_server_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, RemoteServerBillingSession, ParamT], HttpResponse],
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest,
-        /,
-        *args: ParamT.args,
-        **kwargs: ParamT.kwargs,
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-
-        server_uuid = kwargs.get("server_uuid")
-        if not isinstance(server_uuid, str):
-            raise TypeError("server_uuid must be a string")  # nocoverage
-
-        try:
-            remote_server, remote_billing_user = get_remote_server_and_user_from_session(
-                request, server_uuid=server_uuid
-            )
-            if remote_billing_user is None:
-                # This should only be possible if the user hasn't finished the confirmation flow
-                # and doesn't have a fully authenticated session yet. They should not be attempting
-                # to access this endpoint yet.
-                raise RemoteBillingAuthenticationError
-        except (RemoteBillingIdentityExpiredError, RemoteBillingAuthenticationError):
-            # In this flow, we can only redirect to our local "legacy server flow login" page.
-            # That means that we can do it universally whether the user has an expired
-            # identity_dict, or just lacks any form of authentication info at all - there
-            # are no security concerns since this is just a local redirect.
-            url = reverse("remote_billing_legacy_server_login")
-            page_type = get_next_page_param_from_request_path(request)
-            if page_type is not None:
-                query = urlencode({"next_page": page_type})
-                url = append_url_query_string(url, query)
-
-            # Return error for AJAX requests with url.
-            if request.headers.get("x-requested-with") == "XMLHttpRequest":  # nocoverage
-                return session_expired_ajax_response(url)
-
-            return HttpResponseRedirect(url)
-
-        assert remote_billing_user is not None
-        billing_session = RemoteServerBillingSession(
-            remote_server, remote_billing_user=remote_billing_user
-        )
-        return view_func(request, billing_session)
-
-    return _wrapped_view_func

corporate/lib/registration.py

@@ -1,15 +1,16 @@
+from typing import Optional
+
 from django.conf import settings
 from django.utils.translation import gettext as _
 
 from corporate.lib.stripe import LicenseLimitError, get_latest_seat_count, get_seat_count
 from corporate.models import get_current_plan_by_realm
-from zerver.actions.create_user import send_group_direct_message_to_admins
+from zerver.actions.create_user import send_message_to_signup_notification_stream
 from zerver.lib.exceptions import InvitationError
-from zerver.models import Realm, UserProfile
-from zerver.models.users import get_system_bot
+from zerver.models import Realm, UserProfile, get_system_bot
 
 
-def generate_licenses_low_warning_message_if_required(realm: Realm) -> str | None:
+def generate_licenses_low_warning_message_if_required(realm: Realm) -> Optional[str]:
     plan = get_current_plan_by_realm(realm)
     if plan is None or plan.automanage_licenses:
         return None
@@ -49,7 +50,7 @@ def generate_licenses_low_warning_message_if_required(realm: Realm) -> str | Non
     }[licenses_remaining].format(**format_kwargs)
 
 
-def send_user_unable_to_signup_group_direct_message_to_admins(
+def send_user_unable_to_signup_message_to_signup_notification_stream(
     realm: Realm, user_email: str
 ) -> None:
     message = _(
@@ -62,7 +63,7 @@ def send_user_unable_to_signup_group_direct_message_to_admins(
         deactivate_user_help_page_link="/help/deactivate-or-reactivate-a-user",
     )
 
-    send_group_direct_message_to_admins(
+    send_message_to_signup_notification_stream(
         get_system_bot(settings.NOTIFICATION_BOT, realm.id), realm, message
     )
 
@@ -91,7 +92,7 @@ def check_spare_licenses_available_for_registering_new_user(
         else:
             check_spare_licenses_available_for_adding_new_users(realm, extra_non_guests_count=1)
     except LicenseLimitError:
-        send_user_unable_to_signup_group_direct_message_to_admins(realm, user_email_to_add)
+        send_user_unable_to_signup_message_to_signup_notification_stream(realm, user_email_to_add)
         raise
 
 

corporate/lib/remote_billing_util.py

@@ -1,182 +0,0 @@
-import logging
-from typing import Literal, TypedDict, cast
-
-from django.http import HttpRequest
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-
-from zerver.lib.exceptions import JsonableError, RemoteBillingAuthenticationError
-from zerver.lib.timestamp import datetime_to_timestamp
-from zilencer.models import (
-    RemoteRealm,
-    RemoteRealmBillingUser,
-    RemoteServerBillingUser,
-    RemoteZulipServer,
-)
-
-billing_logger = logging.getLogger("corporate.stripe")
-
-# The sessions are relatively short-lived, so that we can avoid issues
-# with users who have their privileges revoked on the remote server
-# maintaining access to the billing page for too long.
-REMOTE_BILLING_SESSION_VALIDITY_SECONDS = 2 * 60 * 60
-
-
-class RemoteBillingUserDict(TypedDict):
-    user_uuid: str
-    user_email: str
-    user_full_name: str
-
-
-class RemoteBillingIdentityDict(TypedDict):
-    user: RemoteBillingUserDict
-    remote_server_uuid: str
-    remote_realm_uuid: str
-
-    remote_billing_user_id: int | None
-    authenticated_at: int
-    uri_scheme: Literal["http://", "https://"]
-
-    next_page: str | None
-
-
-class LegacyServerIdentityDict(TypedDict):
-    # Currently this has only one field. We can extend this
-    # to add more information as appropriate.
-    remote_server_uuid: str
-
-    remote_billing_user_id: int | None
-    authenticated_at: int
-
-
-class RemoteBillingIdentityExpiredError(Exception):
-    def __init__(
-        self,
-        *,
-        realm_uuid: str | None = None,
-        server_uuid: str | None = None,
-        uri_scheme: Literal["http://", "https://"] | None = None,
-    ) -> None:
-        self.realm_uuid = realm_uuid
-        self.server_uuid = server_uuid
-        self.uri_scheme = uri_scheme
-
-
-def get_identity_dict_from_session(
-    request: HttpRequest,
-    *,
-    realm_uuid: str | None,
-    server_uuid: str | None,
-) -> RemoteBillingIdentityDict | LegacyServerIdentityDict | None:
-    if not (realm_uuid or server_uuid):
-        return None
-
-    identity_dicts = request.session.get("remote_billing_identities")
-    if identity_dicts is None:
-        return None
-
-    if realm_uuid is not None:
-        result = identity_dicts.get(f"remote_realm:{realm_uuid}")
-    else:
-        assert server_uuid is not None
-        result = identity_dicts.get(f"remote_server:{server_uuid}")
-
-    if result is None:
-        return None
-    if (
-        datetime_to_timestamp(timezone_now()) - result["authenticated_at"]
-        > REMOTE_BILLING_SESSION_VALIDITY_SECONDS
-    ):
-        # In this case we raise, because callers want to catch this as an explicitly
-        # different scenario from the user not being authenticated, to handle it nicely
-        # by redirecting them to their login page.
-        raise RemoteBillingIdentityExpiredError(
-            realm_uuid=result.get("remote_realm_uuid"),
-            server_uuid=result.get("remote_server_uuid"),
-            uri_scheme=result.get("uri_scheme"),
-        )
-
-    return result
-
-
-def get_remote_realm_and_user_from_session(
-    request: HttpRequest,
-    realm_uuid: str | None,
-) -> tuple[RemoteRealm, RemoteRealmBillingUser]:
-    # Cannot use isinstance with TypeDicts, to make mypy know
-    # which of the TypedDicts in the Union this is - so just cast it.
-    identity_dict = cast(
-        RemoteBillingIdentityDict | None,
-        get_identity_dict_from_session(request, realm_uuid=realm_uuid, server_uuid=None),
-    )
-
-    if identity_dict is None:
-        raise RemoteBillingAuthenticationError
-
-    remote_server_uuid = identity_dict["remote_server_uuid"]
-    remote_realm_uuid = identity_dict["remote_realm_uuid"]
-
-    try:
-        remote_realm = RemoteRealm.objects.get(
-            uuid=remote_realm_uuid, server__uuid=remote_server_uuid
-        )
-    except RemoteRealm.DoesNotExist:
-        raise AssertionError(
-            "The remote realm is missing despite being in the RemoteBillingIdentityDict"
-        )
-
-    if (
-        remote_realm.registration_deactivated
-        or remote_realm.realm_deactivated
-        or remote_realm.server.deactivated
-    ):
-        raise JsonableError(_("Registration is deactivated"))
-
-    remote_billing_user_id = identity_dict["remote_billing_user_id"]
-    # We only put IdentityDicts with remote_billing_user_id in the session in this flow,
-    # because the RemoteRealmBillingUser already exists when this is inserted into the session
-    # at the end of authentication.
-    assert remote_billing_user_id is not None
-
-    try:
-        remote_billing_user = RemoteRealmBillingUser.objects.get(
-            id=remote_billing_user_id, remote_realm=remote_realm
-        )
-    except RemoteRealmBillingUser.DoesNotExist:
-        raise AssertionError
-
-    return remote_realm, remote_billing_user
-
-
-def get_remote_server_and_user_from_session(
-    request: HttpRequest,
-    server_uuid: str,
-) -> tuple[RemoteZulipServer, RemoteServerBillingUser | None]:
-    identity_dict: LegacyServerIdentityDict | None = get_identity_dict_from_session(
-        request, realm_uuid=None, server_uuid=server_uuid
-    )
-
-    if identity_dict is None:
-        raise RemoteBillingAuthenticationError
-
-    remote_server_uuid = identity_dict["remote_server_uuid"]
-    try:
-        remote_server = RemoteZulipServer.objects.get(uuid=remote_server_uuid)
-    except RemoteZulipServer.DoesNotExist:
-        raise JsonableError(_("Invalid remote server."))
-
-    if remote_server.deactivated:
-        raise JsonableError(_("Registration is deactivated"))
-
-    remote_billing_user_id = identity_dict.get("remote_billing_user_id")
-    if remote_billing_user_id is None:
-        return remote_server, None
-
-    try:
-        remote_billing_user = RemoteServerBillingUser.objects.get(
-            id=remote_billing_user_id, remote_server=remote_server
-        )
-    except RemoteServerBillingUser.DoesNotExist:
-        remote_billing_user = None
-
-    return remote_server, remote_billing_user

corporate/lib/stripe_event_handler.py

@@ -1,38 +1,28 @@
 import logging
-from collections.abc import Callable
-from typing import Any
+from contextlib import suppress
+from typing import Any, Callable, Dict, Union
 
 import stripe
 from django.conf import settings
 
 from corporate.lib.stripe import (
-    BILLING_SUPPORT_EMAIL,
     BillingError,
-    RealmBillingSession,
-    RemoteRealmBillingSession,
-    RemoteServerBillingSession,
-    get_configured_fixed_price_plan_offer,
+    UpgradeWithExistingPlanError,
+    ensure_realm_does_not_have_active_plan,
+    process_initial_upgrade,
+    update_or_create_stripe_customer,
 )
-from corporate.models import (
-    Customer,
-    CustomerPlan,
-    Event,
-    Invoice,
-    Session,
-    get_current_plan_by_customer,
-)
-from zerver.lib.send_email import FromAddress, send_email
-from zerver.models.users import get_active_user_profile_by_id_in_realm
+from corporate.models import Event, PaymentIntent, Session
+from zerver.models import get_active_user_profile_by_id_in_realm
 
 billing_logger = logging.getLogger("corporate.stripe")
 
 
-def stripe_event_handler_decorator(
+def error_handler(
     func: Callable[[Any, Any], None],
-) -> Callable[[stripe.checkout.Session | stripe.Invoice, Event], None]:
+) -> Callable[[Union[stripe.checkout.Session, stripe.PaymentIntent], Event], None]:
     def wrapper(
-        stripe_object: stripe.checkout.Session | stripe.Invoice,
-        event: Event,
+        stripe_object: Union[stripe.checkout.Session, stripe.PaymentIntent], event: Event
     ) -> None:
         event.status = Event.EVENT_HANDLER_STARTED
         event.save(update_fields=["status"])
@@ -40,7 +30,7 @@ def stripe_event_handler_decorator(
         try:
             func(stripe_object, event.content_object)
         except BillingError as e:
-            message = (
+            billing_logger.warning(
                 "BillingError in %s event handler: %s. stripe_object_id=%s, customer_id=%s metadata=%s",
                 event.type,
                 e.error_description,
@@ -48,23 +38,12 @@ def stripe_event_handler_decorator(
                 stripe_object.customer,
                 stripe_object.metadata,
             )
-            billing_logger.warning(message)
             event.status = Event.EVENT_HANDLER_FAILED
             event.handler_error = {
                 "message": e.msg,
                 "description": e.error_description,
             }
             event.save(update_fields=["status", "handler_error"])
-            if type(stripe_object) == stripe.Invoice:
-                # For Invoice processing errors, send email to billing support.
-                send_email(
-                    "zerver/emails/error_processing_invoice",
-                    to_emails=[BILLING_SUPPORT_EMAIL],
-                    from_address=FromAddress.tokenized_no_reply_address(),
-                    context={
-                        "message": message,
-                    },
-                )
         except Exception:
             billing_logger.exception(
                 "Uncaught exception in %s event handler:",
@@ -84,127 +63,120 @@ def stripe_event_handler_decorator(
     return wrapper
 
 
-def get_billing_session_for_stripe_webhook(
-    customer: Customer, user_id: str | None
-) -> RealmBillingSession | RemoteRealmBillingSession | RemoteServerBillingSession:
-    if customer.remote_realm is not None:  # nocoverage
-        return RemoteRealmBillingSession(customer.remote_realm)
-    elif customer.remote_server is not None:  # nocoverage
-        return RemoteServerBillingSession(customer.remote_server)
-    else:
-        assert user_id is not None
-        assert customer.realm is not None
-        user = get_active_user_profile_by_id_in_realm(int(user_id), customer.realm)
-        return RealmBillingSession(user)
-
-
-@stripe_event_handler_decorator
+@error_handler
 def handle_checkout_session_completed_event(
     stripe_session: stripe.checkout.Session, session: Session
 ) -> None:
     session.status = Session.COMPLETED
     session.save()
 
-    assert isinstance(stripe_session.setup_intent, str)
-    assert stripe_session.metadata is not None
     stripe_setup_intent = stripe.SetupIntent.retrieve(stripe_session.setup_intent)
-    billing_session = get_billing_session_for_stripe_webhook(
-        session.customer, stripe_session.metadata.get("user_id")
-    )
+    assert session.customer.realm is not None
+    user_id = stripe_session.metadata.get("user_id")
+    assert user_id is not None
+    user = get_active_user_profile_by_id_in_realm(user_id, session.customer.realm)
     payment_method = stripe_setup_intent.payment_method
-    assert isinstance(payment_method, (str, type(None)))  # noqa: UP038  # https://github.com/python/mypy/issues/17413
 
     if session.type in [
-        Session.CARD_UPDATE_FROM_BILLING_PAGE,
-        Session.CARD_UPDATE_FROM_UPGRADE_PAGE,
+        Session.UPGRADE_FROM_BILLING_PAGE,
+        Session.RETRY_UPGRADE_WITH_ANOTHER_PAYMENT_METHOD,
     ]:
-        billing_session.update_or_create_stripe_customer(payment_method)
-
-
-@stripe_event_handler_decorator
-def handle_invoice_paid_event(stripe_invoice: stripe.Invoice, invoice: Invoice) -> None:
-    invoice.status = Invoice.PAID
-    invoice.save(update_fields=["status"])
-
-    customer = invoice.customer
-
-    configured_fixed_price_plan = None
-    if customer.required_plan_tier:
-        configured_fixed_price_plan = get_configured_fixed_price_plan_offer(
-            customer, customer.required_plan_tier
-        )
-
-    if (
-        stripe_invoice.collection_method == "send_invoice"
-        and configured_fixed_price_plan
-        and configured_fixed_price_plan.sent_invoice_id == invoice.stripe_invoice_id
-    ):
-        billing_session = get_billing_session_for_stripe_webhook(customer, user_id=None)
-        remote_server_legacy_plan = billing_session.get_remote_server_legacy_plan(customer)
-        assert customer.required_plan_tier is not None
-        billing_session.process_initial_upgrade(
-            plan_tier=customer.required_plan_tier,
-            # TODO: Currently licenses don't play any role for fixed price plan.
-            # We plan to introduce max_licenses allowed soon.
-            licenses=0,
-            automanage_licenses=True,
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            charge_automatically=False,
-            free_trial=False,
-            remote_server_legacy_plan=remote_server_legacy_plan,
-            stripe_invoice_paid=True,
-        )
-    else:
-        metadata = stripe_invoice.metadata
-        # Only process upgrade required if metadata has the required keys.
-        # This is a safeguard to avoid processing custom invoices.
-        if (
-            metadata is None
-            or metadata.get("billing_schedule") is None
-            or metadata.get("plan_tier") is None
-        ):  # nocoverage
-            return
-
-        billing_session = get_billing_session_for_stripe_webhook(customer, metadata.get("user_id"))
-        remote_server_legacy_plan = billing_session.get_remote_server_legacy_plan(customer)
-        billing_schedule = int(metadata["billing_schedule"])
-        plan_tier = int(metadata["plan_tier"])
-        charge_automatically = stripe_invoice.collection_method != "send_invoice"
-        if configured_fixed_price_plan and customer.required_plan_tier == plan_tier:
-            assert customer.required_plan_tier is not None
-            billing_session.process_initial_upgrade(
-                plan_tier=customer.required_plan_tier,
-                # TODO: Currently licenses don't play any role for fixed price plan.
-                # We plan to introduce max_licenses allowed soon.
-                licenses=0,
-                automanage_licenses=True,
-                billing_schedule=billing_schedule,
-                charge_automatically=charge_automatically,
-                free_trial=False,
-                remote_server_legacy_plan=remote_server_legacy_plan,
-                stripe_invoice_paid=True,
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        assert session.payment_intent is not None
+        session.payment_intent.status = PaymentIntent.PROCESSING
+        session.payment_intent.last_payment_error = ()
+        session.payment_intent.save(update_fields=["status", "last_payment_error"])
+        with suppress(stripe.error.CardError):
+            stripe.PaymentIntent.confirm(
+                session.payment_intent.stripe_payment_intent_id,
+                payment_method=payment_method,
+                off_session=True,
             )
-            return
-        elif metadata.get("on_free_trial") and invoice.is_created_for_free_trial_upgrade:
-            free_trial_plan = invoice.plan
-            assert free_trial_plan is not None
-            if free_trial_plan.is_free_trial():
-                # We don't need to do anything here. When the free trial ends we will
-                # check if user has paid the invoice, if not we downgrade the user.
-                return
-
-            # If customer paid after end of free trial, we just upgrade via default method below.
-            assert free_trial_plan.status == CustomerPlan.ENDED
-            # Also check if customer is not on any other active plan.
-            assert get_current_plan_by_customer(customer) is None
-
-        billing_session.process_initial_upgrade(
-            plan_tier,
-            int(metadata["licenses"]),
-            metadata["license_management"] == "automatic",
-            billing_schedule=billing_schedule,
-            charge_automatically=charge_automatically,
-            free_trial=False,
-            remote_server_legacy_plan=remote_server_legacy_plan,
-            stripe_invoice_paid=True,
+    elif session.type in [
+        Session.FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE,
+        Session.FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE,
+    ]:
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        process_initial_upgrade(
+            user,
+            int(stripe_setup_intent.metadata["licenses"]),
+            stripe_setup_intent.metadata["license_management"] == "automatic",
+            int(stripe_setup_intent.metadata["billing_schedule"]),
+            charge_automatically=True,
+            free_trial=True,
         )
+    elif session.type in [Session.CARD_UPDATE_FROM_BILLING_PAGE]:
+        update_or_create_stripe_customer(user, payment_method)
+
+
+@error_handler
+def handle_payment_intent_succeeded_event(
+    stripe_payment_intent: stripe.PaymentIntent, payment_intent: PaymentIntent
+) -> None:
+    payment_intent.status = PaymentIntent.SUCCEEDED
+    payment_intent.save()
+    metadata: Dict[str, Any] = stripe_payment_intent.metadata
+    assert payment_intent.customer.realm is not None
+    user_id = metadata.get("user_id")
+    assert user_id is not None
+    user = get_active_user_profile_by_id_in_realm(user_id, payment_intent.customer.realm)
+
+    description = ""
+    for charge in stripe_payment_intent.charges:
+        description = f"Payment (Card ending in {charge.payment_method_details.card.last4})"
+        break
+
+    stripe.InvoiceItem.create(
+        amount=stripe_payment_intent.amount * -1,
+        currency="usd",
+        customer=stripe_payment_intent.customer,
+        description=description,
+        discountable=False,
+    )
+    try:
+        ensure_realm_does_not_have_active_plan(user.realm)
+    except UpgradeWithExistingPlanError as e:
+        stripe_invoice = stripe.Invoice.create(
+            auto_advance=True,
+            collection_method="charge_automatically",
+            customer=stripe_payment_intent.customer,
+            days_until_due=None,
+            statement_descriptor="Zulip Cloud Standard Credit",
+        )
+        stripe.Invoice.finalize_invoice(stripe_invoice)
+        raise e
+
+    process_initial_upgrade(
+        user,
+        int(metadata["licenses"]),
+        metadata["license_management"] == "automatic",
+        int(metadata["billing_schedule"]),
+        True,
+        False,
+    )
+
+
+@error_handler
+def handle_payment_intent_payment_failed_event(
+    stripe_payment_intent: stripe.PaymentIntent, payment_intent: PaymentIntent
+) -> None:
+    payment_intent.status = PaymentIntent.get_status_integer_from_status_text(
+        stripe_payment_intent.status
+    )
+    assert payment_intent.customer.realm is not None
+    billing_logger.info(
+        "Stripe payment intent failed: %s %s %s %s",
+        payment_intent.customer.realm.string_id,
+        stripe_payment_intent.last_payment_error.get("type"),
+        stripe_payment_intent.last_payment_error.get("code"),
+        stripe_payment_intent.last_payment_error.get("param"),
+    )
+    payment_intent.last_payment_error = {
+        "description": stripe_payment_intent.last_payment_error.get("type"),
+    }
+    payment_intent.last_payment_error["message"] = stripe_payment_intent.last_payment_error.get(
+        "message"
+    )
+    payment_intent.save(update_fields=["status", "last_payment_error"])

corporate/lib/support.py

@@ -1,428 +1,15 @@
-from dataclasses import dataclass
-from datetime import datetime, timedelta
-from typing import Optional, TypedDict, Union
 from urllib.parse import urlencode, urljoin, urlunsplit
 
 from django.conf import settings
-from django.db.models import Sum
 from django.urls import reverse
-from django.utils.timezone import now as timezone_now
 
-from corporate.lib.stripe import (
-    BillingSession,
-    PushNotificationsEnabledStatus,
-    RealmBillingSession,
-    RemoteRealmBillingSession,
-    RemoteServerBillingSession,
-    get_configured_fixed_price_plan_offer,
-    get_price_per_license,
-    get_push_status_for_remote_request,
-    start_of_next_billing_cycle,
-)
-from corporate.models import (
-    Customer,
-    CustomerPlan,
-    CustomerPlanOffer,
-    ZulipSponsorshipRequest,
-    get_current_plan_by_customer,
-)
-from zerver.models import Realm
-from zerver.models.realms import get_org_type_display_name, get_realm
-from zilencer.lib.remote_counts import MissingDataError
-from zilencer.models import (
-    RemoteCustomerUserCount,
-    RemoteInstallationCount,
-    RemotePushDeviceToken,
-    RemoteRealm,
-    RemoteRealmCount,
-    RemoteZulipServer,
-    RemoteZulipServerAuditLog,
-    get_remote_realm_guest_and_non_guest_count,
-    get_remote_server_guest_and_non_guest_count,
-    has_stale_audit_log,
-)
-
-USER_DATA_STALE_WARNING = "Recent audit log data missing: No information for used licenses"
+from zerver.models import Realm, get_realm
 
 
-class SponsorshipRequestDict(TypedDict):
-    org_type: str
-    org_website: str
-    org_description: str
-    total_users: str
-    paid_users: str
-    paid_users_description: str
-    requested_plan: str
-
-
-@dataclass
-class SponsorshipData:
-    sponsorship_pending: bool = False
-    monthly_discounted_price: int | None = None
-    annual_discounted_price: int | None = None
-    original_monthly_plan_price: int | None = None
-    original_annual_plan_price: int | None = None
-    minimum_licenses: int | None = None
-    required_plan_tier: int | None = None
-    latest_sponsorship_request: SponsorshipRequestDict | None = None
-
-
-@dataclass
-class NextPlanData:
-    plan: Union["CustomerPlan", "CustomerPlanOffer", None] = None
-    estimated_revenue: int | None = None
-
-
-@dataclass
-class PlanData:
-    customer: Optional["Customer"] = None
-    current_plan: Optional["CustomerPlan"] = None
-    next_plan: Union["CustomerPlan", "CustomerPlanOffer", None] = None
-    licenses: int | None = None
-    licenses_used: int | None = None
-    next_billing_cycle_start: datetime | None = None
-    is_legacy_plan: bool = False
-    has_fixed_price: bool = False
-    is_current_plan_billable: bool = False
-    stripe_customer_url: str | None = None
-    warning: str | None = None
-    annual_recurring_revenue: int | None = None
-    estimated_next_plan_revenue: int | None = None
-
-
-@dataclass
-class MobilePushData:
-    total_mobile_users: int
-    push_notification_status: PushNotificationsEnabledStatus
-    uncategorized_mobile_users: int | None = None
-    mobile_pushes_forwarded: int | None = None
-    last_mobile_push_sent: str = ""
-
-
-@dataclass
-class RemoteSupportData:
-    date_created: datetime
-    has_stale_audit_log: bool
-    plan_data: PlanData
-    sponsorship_data: SponsorshipData
-    user_data: RemoteCustomerUserCount
-    mobile_push_data: MobilePushData
-
-
-@dataclass
-class CloudSupportData:
-    plan_data: PlanData
-    sponsorship_data: SponsorshipData
-
-
-def get_stripe_customer_url(stripe_id: str) -> str:
-    return f"https://dashboard.stripe.com/customers/{stripe_id}"  # nocoverage
-
-
-def get_realm_support_url(realm: Realm) -> str:
-    support_realm_url = get_realm(settings.STAFF_SUBDOMAIN).url
+def get_support_url(realm: Realm) -> str:
+    support_realm_uri = get_realm(settings.STAFF_SUBDOMAIN).uri
     support_url = urljoin(
-        support_realm_url,
+        support_realm_uri,
         urlunsplit(("", "", reverse("support"), urlencode({"q": realm.string_id}), "")),
     )
     return support_url
-
-
-def get_customer_sponsorship_data(customer: Customer) -> SponsorshipData:
-    pending = customer.sponsorship_pending
-    licenses = customer.minimum_licenses
-    plan_tier = customer.required_plan_tier
-    sponsorship_request = None
-    monthly_discounted_price = None
-    annual_discounted_price = None
-    original_monthly_plan_price = None
-    original_annual_plan_price = None
-    if customer.monthly_discounted_price:
-        monthly_discounted_price = customer.monthly_discounted_price
-    if customer.annual_discounted_price:
-        annual_discounted_price = customer.annual_discounted_price
-    if plan_tier is not None:
-        original_monthly_plan_price = get_price_per_license(
-            plan_tier, CustomerPlan.BILLING_SCHEDULE_MONTHLY
-        )
-        original_annual_plan_price = get_price_per_license(
-            plan_tier, CustomerPlan.BILLING_SCHEDULE_ANNUAL
-        )
-    if pending:
-        last_sponsorship_request = (
-            ZulipSponsorshipRequest.objects.filter(customer=customer).order_by("id").last()
-        )
-        if last_sponsorship_request is not None:
-            org_type_name = get_org_type_display_name(last_sponsorship_request.org_type)
-            if (
-                last_sponsorship_request.org_website is None
-                or last_sponsorship_request.org_website == ""
-            ):
-                website = "No website submitted"
-            else:
-                website = last_sponsorship_request.org_website
-            sponsorship_request = SponsorshipRequestDict(
-                org_type=org_type_name,
-                org_website=website,
-                org_description=last_sponsorship_request.org_description,
-                total_users=last_sponsorship_request.expected_total_users,
-                paid_users=last_sponsorship_request.paid_users_count,
-                paid_users_description=last_sponsorship_request.paid_users_description,
-                requested_plan=last_sponsorship_request.requested_plan,
-            )
-
-    return SponsorshipData(
-        sponsorship_pending=pending,
-        monthly_discounted_price=monthly_discounted_price,
-        annual_discounted_price=annual_discounted_price,
-        original_monthly_plan_price=original_monthly_plan_price,
-        original_annual_plan_price=original_annual_plan_price,
-        minimum_licenses=licenses,
-        required_plan_tier=plan_tier,
-        latest_sponsorship_request=sponsorship_request,
-    )
-
-
-def get_annual_invoice_count(billing_schedule: int) -> int:
-    if billing_schedule == CustomerPlan.BILLING_SCHEDULE_MONTHLY:
-        return 12
-    else:  # nocoverage
-        return 1
-
-
-def get_next_plan_data(
-    billing_session: BillingSession,
-    customer: Customer,
-    current_plan: CustomerPlan | None = None,
-) -> NextPlanData:
-    plan_offer: CustomerPlanOffer | None = None
-
-    # A customer can have a CustomerPlanOffer with or without a current plan.
-    if customer.required_plan_tier:
-        plan_offer = get_configured_fixed_price_plan_offer(customer, customer.required_plan_tier)
-
-    if plan_offer is not None:
-        next_plan_data = NextPlanData(plan=plan_offer)
-    elif current_plan is not None:
-        next_plan_data = NextPlanData(plan=billing_session.get_next_plan(current_plan))
-    else:
-        next_plan_data = NextPlanData()
-
-    if next_plan_data.plan is not None:
-        if next_plan_data.plan.fixed_price is not None:
-            next_plan_data.estimated_revenue = next_plan_data.plan.fixed_price
-            return next_plan_data
-
-        if current_plan is not None:
-            licenses_at_next_renewal = current_plan.licenses_at_next_renewal()
-            if licenses_at_next_renewal is not None:
-                assert type(next_plan_data.plan) is CustomerPlan
-                assert next_plan_data.plan.price_per_license is not None
-                invoice_count = get_annual_invoice_count(next_plan_data.plan.billing_schedule)
-                next_plan_data.estimated_revenue = (
-                    next_plan_data.plan.price_per_license * licenses_at_next_renewal * invoice_count
-                )
-            else:
-                next_plan_data.estimated_revenue = 0  # nocoverage
-            return next_plan_data
-
-    return next_plan_data
-
-
-def get_plan_data_for_support_view(
-    billing_session: BillingSession, user_count: int | None = None, stale_user_data: bool = False
-) -> PlanData:
-    customer = billing_session.get_customer()
-    plan = None
-    if customer is not None:
-        plan = get_current_plan_by_customer(customer)
-    plan_data = PlanData(
-        customer=customer,
-        current_plan=plan,
-    )
-
-    if plan is not None:
-        new_plan, last_ledger_entry = billing_session.make_end_of_cycle_updates_if_needed(
-            plan, timezone_now()
-        )
-        if last_ledger_entry is not None:
-            if new_plan is not None:
-                plan_data.current_plan = new_plan  # nocoverage
-            plan_data.licenses = last_ledger_entry.licenses
-        assert plan_data.current_plan is not None  # for mypy
-
-        # If we already have user count data, we use that
-        # instead of querying the database again to get
-        # the number of currently used licenses.
-        if stale_user_data:
-            plan_data.warning = USER_DATA_STALE_WARNING
-        elif user_count is None:
-            try:
-                plan_data.licenses_used = billing_session.current_count_for_billed_licenses()
-            except MissingDataError:  # nocoverage
-                plan_data.warning = USER_DATA_STALE_WARNING
-        else:  # nocoverage
-            assert user_count is not None
-            plan_data.licenses_used = user_count
-
-        if plan_data.current_plan.status in (
-            CustomerPlan.FREE_TRIAL,
-            CustomerPlan.DOWNGRADE_AT_END_OF_FREE_TRIAL,
-        ):  # nocoverage
-            assert plan_data.current_plan.next_invoice_date is not None
-            plan_data.next_billing_cycle_start = plan_data.current_plan.next_invoice_date
-        else:
-            plan_data.next_billing_cycle_start = start_of_next_billing_cycle(
-                plan_data.current_plan, timezone_now()
-            )
-
-        plan_data.is_legacy_plan = (
-            plan_data.current_plan.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY
-        )
-        plan_data.has_fixed_price = plan_data.current_plan.fixed_price is not None
-        plan_data.is_current_plan_billable = billing_session.check_plan_tier_is_billable(
-            plan_tier=plan_data.current_plan.tier
-        )
-        if last_ledger_entry is not None:
-            plan_data.annual_recurring_revenue = (
-                billing_session.get_annual_recurring_revenue_for_support_data(
-                    plan_data.current_plan, last_ledger_entry
-                )
-            )
-        else:
-            plan_data.annual_recurring_revenue = 0  # nocoverage
-
-    # Check for a non-active/scheduled CustomerPlan or CustomerPlanOffer
-    if customer is not None:
-        next_plan_data = get_next_plan_data(billing_session, customer, plan_data.current_plan)
-        plan_data.next_plan = next_plan_data.plan
-        plan_data.estimated_next_plan_revenue = next_plan_data.estimated_revenue
-
-    # If customer has a stripe ID, add link to stripe customer dashboard
-    if customer is not None and customer.stripe_customer_id is not None:
-        plan_data.stripe_customer_url = get_stripe_customer_url(
-            customer.stripe_customer_id
-        )  # nocoverage
-
-    return plan_data
-
-
-def get_mobile_push_data(remote_entity: RemoteZulipServer | RemoteRealm) -> MobilePushData:
-    if isinstance(remote_entity, RemoteZulipServer):
-        total_users = (
-            RemotePushDeviceToken.objects.filter(server=remote_entity)
-            .distinct("user_id", "user_uuid")
-            .count()
-        )
-        uncategorized_users = (
-            RemotePushDeviceToken.objects.filter(server=remote_entity, remote_realm__isnull=True)
-            .distinct("user_id", "user_uuid")
-            .count()
-        )
-        mobile_pushes = RemoteInstallationCount.objects.filter(
-            server=remote_entity,
-            property="mobile_pushes_forwarded::day",
-            end_time__gte=timezone_now() - timedelta(days=7),
-        ).aggregate(total_forwarded=Sum("value", default=0))
-        latest_remote_server_push_forwarded_count = RemoteInstallationCount.objects.filter(
-            server=remote_entity,
-            property="mobile_pushes_forwarded::day",
-        ).last()
-        if latest_remote_server_push_forwarded_count is not None:  # nocoverage
-            # mobile_pushes_forwarded is a CountStat with a day frequency,
-            # so we want to show the start of the latest day interval.
-            push_forwarded_interval_start = (
-                latest_remote_server_push_forwarded_count.end_time - timedelta(days=1)
-            ).strftime("%Y-%m-%d")
-        else:
-            push_forwarded_interval_start = "None"
-        push_notification_status = get_push_status_for_remote_request(
-            remote_server=remote_entity, remote_realm=None
-        )
-        return MobilePushData(
-            total_mobile_users=total_users,
-            push_notification_status=push_notification_status,
-            uncategorized_mobile_users=uncategorized_users,
-            mobile_pushes_forwarded=mobile_pushes["total_forwarded"],
-            last_mobile_push_sent=push_forwarded_interval_start,
-        )
-    else:
-        assert isinstance(remote_entity, RemoteRealm)
-        mobile_users = (
-            RemotePushDeviceToken.objects.filter(remote_realm=remote_entity)
-            .distinct("user_id", "user_uuid")
-            .count()
-        )
-        mobile_pushes = RemoteRealmCount.objects.filter(
-            remote_realm=remote_entity,
-            property="mobile_pushes_forwarded::day",
-            end_time__gte=timezone_now() - timedelta(days=7),
-        ).aggregate(total_forwarded=Sum("value", default=0))
-        latest_remote_realm_push_forwarded_count = RemoteRealmCount.objects.filter(
-            remote_realm=remote_entity,
-            property="mobile_pushes_forwarded::day",
-        ).last()
-        if latest_remote_realm_push_forwarded_count is not None:  # nocoverage
-            # mobile_pushes_forwarded is a CountStat with a day frequency,
-            # so we want to show the start of the latest day interval.
-            push_forwarded_interval_start = (
-                latest_remote_realm_push_forwarded_count.end_time - timedelta(days=1)
-            ).strftime("%Y-%m-%d")
-        else:
-            push_forwarded_interval_start = "None"
-        push_notification_status = get_push_status_for_remote_request(
-            remote_entity.server, remote_entity
-        )
-        return MobilePushData(
-            total_mobile_users=mobile_users,
-            push_notification_status=push_notification_status,
-            uncategorized_mobile_users=None,
-            mobile_pushes_forwarded=mobile_pushes["total_forwarded"],
-            last_mobile_push_sent=push_forwarded_interval_start,
-        )
-
-
-def get_data_for_remote_support_view(billing_session: BillingSession) -> RemoteSupportData:
-    if isinstance(billing_session, RemoteServerBillingSession):
-        user_data = get_remote_server_guest_and_non_guest_count(billing_session.remote_server.id)
-        stale_audit_log_data = has_stale_audit_log(billing_session.remote_server)
-        date_created = RemoteZulipServerAuditLog.objects.get(
-            event_type=RemoteZulipServerAuditLog.REMOTE_SERVER_CREATED,
-            server__id=billing_session.remote_server.id,
-        ).event_time
-        mobile_data = get_mobile_push_data(billing_session.remote_server)
-    else:
-        assert isinstance(billing_session, RemoteRealmBillingSession)
-        user_data = get_remote_realm_guest_and_non_guest_count(billing_session.remote_realm)
-        stale_audit_log_data = has_stale_audit_log(billing_session.remote_realm.server)
-        date_created = billing_session.remote_realm.realm_date_created
-        mobile_data = get_mobile_push_data(billing_session.remote_realm)
-    user_count = user_data.guest_user_count + user_data.non_guest_user_count
-    plan_data = get_plan_data_for_support_view(billing_session, user_count, stale_audit_log_data)
-    if plan_data.customer is not None:
-        sponsorship_data = get_customer_sponsorship_data(plan_data.customer)
-    else:
-        sponsorship_data = SponsorshipData()
-
-    return RemoteSupportData(
-        date_created=date_created,
-        has_stale_audit_log=stale_audit_log_data,
-        plan_data=plan_data,
-        sponsorship_data=sponsorship_data,
-        user_data=user_data,
-        mobile_push_data=mobile_data,
-    )
-
-
-def get_data_for_cloud_support_view(billing_session: BillingSession) -> CloudSupportData:
-    assert isinstance(billing_session, RealmBillingSession)
-    plan_data = get_plan_data_for_support_view(billing_session)
-    if plan_data.customer is not None:
-        sponsorship_data = get_customer_sponsorship_data(plan_data.customer)
-    else:
-        sponsorship_data = SponsorshipData()
-
-    return CloudSupportData(
-        plan_data=plan_data,
-        sponsorship_data=sponsorship_data,
-    )

corporate/migrations/0019_zulipsponsorshiprequest_expected_total_users_and_more.py

@@ -1,27 +0,0 @@
-# Generated by Django 4.2.6 on 2023-11-11 14:16
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0018_customer_cloud_xor_self_hosted"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="zulipsponsorshiprequest",
-            name="expected_total_users",
-            field=models.TextField(default=""),
-        ),
-        migrations.AddField(
-            model_name="zulipsponsorshiprequest",
-            name="paid_users_count",
-            field=models.TextField(default=""),
-        ),
-        migrations.AddField(
-            model_name="zulipsponsorshiprequest",
-            name="paid_users_description",
-            field=models.TextField(default=""),
-        ),
-    ]

corporate/migrations/0020_add_remote_realm_customers.py

@@ -1,37 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-17 20:11
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("zilencer", "0035_remoterealmcount_remote_realm_and_more"),
-        ("corporate", "0019_zulipsponsorshiprequest_expected_total_users_and_more"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="customer",
-            name="cloud_xor_self_hosted",
-        ),
-        migrations.AddField(
-            model_name="customer",
-            name="remote_realm",
-            field=models.OneToOneField(
-                null=True, on_delete=django.db.models.deletion.CASCADE, to="zilencer.remoterealm"
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="customer",
-            constraint=models.CheckConstraint(
-                check=models.Q(
-                    ("realm__isnull", False),
-                    ("remote_server__isnull", False),
-                    ("remote_realm__isnull", False),
-                    _connector="OR",
-                ),
-                name="has_associated_model_object",
-            ),
-        ),
-    ]

corporate/migrations/0021_remove_session_payment_intent.py

@@ -1,16 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-18 14:54
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0020_add_remote_realm_customers"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="session",
-            name="payment_intent",
-        ),
-    ]

corporate/migrations/0022_session_is_manual_license_management_upgrade_session.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-21 11:20
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0021_remove_session_payment_intent"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="session",
-            name="is_manual_license_management_upgrade_session",
-            field=models.BooleanField(default=False),
-        ),
-    ]

corporate/migrations/0023_zulipsponsorshiprequest_customer.py

@@ -1,20 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-26 16:00
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0022_session_is_manual_license_management_upgrade_session"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="zulipsponsorshiprequest",
-            name="customer",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
-            ),
-        ),
-    ]

corporate/migrations/0024_zulipsponsorshiprequest_fill_customer_data.py

@@ -1,18 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0023_zulipsponsorshiprequest_customer"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            """
-            UPDATE corporate_zulipsponsorshiprequest
-            SET customer_id = (
-                SELECT id FROM corporate_customer WHERE corporate_customer.realm_id = corporate_zulipsponsorshiprequest.realm_id
-            )
-        """
-        ),
-    ]

corporate/migrations/0025_alter_zulipsponsorshiprequest_customer.py

@@ -1,20 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-26 16:13
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0024_zulipsponsorshiprequest_fill_customer_data"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="zulipsponsorshiprequest",
-            name="customer",
-            field=models.ForeignKey(
-                on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
-            ),
-        ),
-    ]

corporate/migrations/0026_remove_zulipsponsorshiprequest_realm.py

@@ -1,16 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-26 16:14
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0025_alter_zulipsponsorshiprequest_customer"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="zulipsponsorshiprequest",
-            name="realm",
-        ),
-    ]

corporate/migrations/0027_alter_zulipsponsorshiprequest_requested_by.py

@@ -1,25 +0,0 @@
-# Generated by Django 4.2.7 on 2023-11-28 16:00
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ("corporate", "0026_remove_zulipsponsorshiprequest_realm"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="zulipsponsorshiprequest",
-            name="requested_by",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                to=settings.AUTH_USER_MODEL,
-            ),
-        ),
-    ]

corporate/migrations/0028_zulipsponsorshiprequest_requested_plan.py

@@ -1,21 +0,0 @@
-# Generated by Django 4.2.8 on 2023-12-08 18:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0027_alter_zulipsponsorshiprequest_requested_by"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="zulipsponsorshiprequest",
-            name="requested_plan",
-            field=models.CharField(
-                choices=[("", "UNSPECIFIED"), ("Community", "COMMUNITY"), ("Business", "BUSINESS")],
-                default="",
-                max_length=50,
-            ),
-        ),
-    ]

corporate/migrations/0029_session_tier.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.8 on 2023-12-18 09:39
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0028_zulipsponsorshiprequest_requested_plan"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="session",
-            name="tier",
-            field=models.SmallIntegerField(null=True),
-        ),
-    ]

corporate/migrations/0030_alter_zulipsponsorshiprequest_requested_plan.py

@@ -1,26 +0,0 @@
-# Generated by Django 4.2.8 on 2023-12-19 11:38
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0029_session_tier"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="zulipsponsorshiprequest",
-            name="requested_plan",
-            field=models.CharField(
-                choices=[
-                    ("", "UNSPECIFIED"),
-                    ("Community", "COMMUNITY"),
-                    ("Basic", "BASIC"),
-                    ("Business", "BUSINESS"),
-                ],
-                default="",
-                max_length=50,
-            ),
-        ),
-    ]

corporate/migrations/0031_customer_flat_discount_and_more.py

@@ -1,22 +0,0 @@
-# Generated by Django 4.2.8 on 2023-12-19 12:24
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0030_alter_zulipsponsorshiprequest_requested_plan"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customer",
-            name="flat_discount",
-            field=models.IntegerField(default=2000),
-        ),
-        migrations.AddField(
-            model_name="customer",
-            name="flat_discounted_months",
-            field=models.IntegerField(default=0),
-        ),
-    ]

corporate/migrations/0032_customer_minimum_licenses.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.8 on 2023-12-14 13:27
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0031_customer_flat_discount_and_more"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customer",
-            name="minimum_licenses",
-            field=models.PositiveIntegerField(null=True),
-        ),
-    ]

corporate/migrations/0033_customerplan_invoice_overdue_email_sent.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-10 07:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0032_customer_minimum_licenses"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customerplan",
-            name="invoice_overdue_email_sent",
-            field=models.BooleanField(default=False),
-        ),
-    ]

corporate/migrations/0034_customer_discount_required_tier.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-09 14:28
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0033_customerplan_invoice_overdue_email_sent"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customer",
-            name="required_plan_tier",
-            field=models.SmallIntegerField(null=True),
-        ),
-    ]

corporate/migrations/0035_update_legacy_plan_next_invoice_date.py

@@ -1,28 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-25 05:49
-
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db.models import F
-
-
-def update_legacy_plan_next_invoice_date(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    CustomerPlan = apps.get_model("corporate", "CustomerPlan")
-    CustomerPlan.TIER_SELF_HOSTED_LEGACY = 101
-
-    # For legacy plans, set next_invoice_date = end_date.
-    CustomerPlan.objects.filter(tier=CustomerPlan.TIER_SELF_HOSTED_LEGACY).update(
-        next_invoice_date=F("end_date")
-    )
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0034_customer_discount_required_tier"),
-    ]
-
-    operations = [
-        migrations.RunPython(update_legacy_plan_next_invoice_date),
-    ]

corporate/migrations/0036_fix_customer_plans_scheduled_after_legacy_plan.py

@@ -1,40 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-24 08:08
-
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def fix_customer_plans_scheduled_after_legacy_plan(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    CustomerPlan = apps.get_model("corporate", "CustomerPlan")
-    CustomerPlan.TIER_SELF_HOSTED_LEGACY = 101
-    CustomerPlan.SWITCH_PLAN_TIER_AT_PLAN_END = 8
-    CustomerPlan.NEVER_STARTED = 12
-    CustomerPlan.INVOICING_STATUS_INITIAL_INVOICE_TO_BE_SENT = 3
-
-    # Legacy plans scheduled to switch to a new plan.
-    legacy_plans = CustomerPlan.objects.filter(
-        tier=CustomerPlan.TIER_SELF_HOSTED_LEGACY, status=CustomerPlan.SWITCH_PLAN_TIER_AT_PLAN_END
-    )
-
-    for legacy_plan in legacy_plans:
-        CustomerPlan.objects.filter(
-            customer=legacy_plan.customer,
-            billing_cycle_anchor=legacy_plan.end_date,
-            status=CustomerPlan.NEVER_STARTED,
-        ).update(
-            next_invoice_date=legacy_plan.end_date,
-            invoicing_status=CustomerPlan.INVOICING_STATUS_INITIAL_INVOICE_TO_BE_SENT,
-        )
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0035_update_legacy_plan_next_invoice_date"),
-    ]
-
-    operations = [
-        migrations.RunPython(fix_customer_plans_scheduled_after_legacy_plan),
-    ]

corporate/migrations/0037_customerplanoffer.py

@@ -1,36 +0,0 @@
-# Generated by Django 4.2.8 on 2024-01-27 09:03
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0036_fix_customer_plans_scheduled_after_legacy_plan"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="CustomerPlanOffer",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                ("fixed_price", models.IntegerField(null=True)),
-                ("tier", models.SmallIntegerField()),
-                ("status", models.SmallIntegerField()),
-                (
-                    "customer",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-            },
-        ),
-    ]

corporate/migrations/0038_customerplanoffer_sent_invoice_id_invoice.py

@@ -1,37 +0,0 @@
-# Generated by Django 4.2.9 on 2024-02-02 10:15
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0037_customerplanoffer"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customerplanoffer",
-            name="sent_invoice_id",
-            field=models.CharField(max_length=255, null=True),
-        ),
-        migrations.CreateModel(
-            name="Invoice",
-            fields=[
-                (
-                    "id",
-                    models.AutoField(
-                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-                    ),
-                ),
-                ("stripe_invoice_id", models.CharField(max_length=255, unique=True)),
-                ("status", models.SmallIntegerField()),
-                (
-                    "customer",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="corporate.customer"
-                    ),
-                ),
-            ],
-        ),
-    ]

corporate/migrations/0039_backfill_end_date_for_fixed_price_plans.py

@@ -1,62 +0,0 @@
-# Generated by Django 4.2.9 on 2024-02-21 11:53
-
-from datetime import datetime
-
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def add_months(dt: datetime, months: int) -> datetime:
-    assert months >= 0
-    # It's fine that the max day in Feb is 28 for leap years.
-    MAX_DAY_FOR_MONTH = {
-        1: 31,
-        2: 28,
-        3: 31,
-        4: 30,
-        5: 31,
-        6: 30,
-        7: 31,
-        8: 31,
-        9: 30,
-        10: 31,
-        11: 30,
-        12: 31,
-    }
-    year = dt.year
-    month = dt.month + months
-    while month > 12:
-        year += 1
-        month -= 12
-    day = min(dt.day, MAX_DAY_FOR_MONTH[month])
-    # datetimes don't support leap seconds, so don't need to worry about those
-    return dt.replace(year=year, month=month, day=day)
-
-
-def backfill_end_date_for_fixed_price_plans(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    CustomerPlan = apps.get_model("corporate", "CustomerPlan")
-    CustomerPlan.ACTIVE = 1
-
-    fixed_price_plans = CustomerPlan.objects.filter(
-        status=CustomerPlan.ACTIVE, fixed_price__isnull=False
-    )
-    for plan in fixed_price_plans:
-        assert plan.end_date is None
-        assert plan.billing_cycle_anchor is not None
-        plan.end_date = add_months(plan.billing_cycle_anchor, 12)
-        plan.save(update_fields=["end_date"])
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0038_customerplanoffer_sent_invoice_id_invoice"),
-    ]
-
-    operations = [
-        migrations.RunPython(
-            backfill_end_date_for_fixed_price_plans,
-        ),
-    ]

corporate/migrations/0040_customerplan_reminder_to_review_plan_email_sent.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2.9 on 2024-02-15 06:47
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0039_backfill_end_date_for_fixed_price_plans"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="customerplan",
-            name="reminder_to_review_plan_email_sent",
-            field=models.BooleanField(default=False),
-        ),
-    ]