Release Zulip Server 6.0-rc1.

.codespellignore

@@ -21,7 +21,3 @@ vise
 falsy
 ro
 derails
-forin
-uper
-slac
-couldn

.eslintignore

@@ -4,12 +4,11 @@
 
 /docs/_build
 /static/generated
+/static/third
 /static/webpack-bundles
 /var/*
 !/var/puppeteer
 /var/puppeteer/*
 !/var/puppeteer/test_credentials.d.ts
-/web/generated
-/web/third
 /zulip-current-venv
 /zulip-py3-venv

.eslintrc.json

@@ -1,5 +1,4 @@
 {
-    "root": true,
     "env": {
         "es2020": true,
         "node": true
@@ -15,7 +14,6 @@
     ],
     "parser": "@babel/eslint-parser",
     "parserOptions": {
-        "requireConfigFile": false,
         "warnOnUnsupportedTypeScriptVersion": false,
         "sourceType": "unambiguous"
     },
@@ -50,12 +48,15 @@
         "import/extensions": "error",
         "import/first": "error",
         "import/newline-after-import": "error",
-        "import/no-cycle": ["error", {"ignoreExternal": true}],
-        "import/no-duplicates": "error",
         "import/no-self-import": "error",
-        "import/no-unresolved": "off",
         "import/no-useless-path-segments": "error",
-        "import/order": ["error", {"alphabetize": {"order": "asc"}, "newlines-between": "always"}],
+        "import/order": [
+            "error",
+            {
+                "alphabetize": {"order": "asc"},
+                "newlines-between": "always"
+            }
+        ],
         "import/unambiguous": "error",
         "lines-around-directive": "error",
         "new-cap": "error",
@@ -66,6 +67,7 @@
         "no-catch-shadow": "error",
         "no-constant-condition": ["error", {"checkLoops": false}],
         "no-div-regex": "error",
+        "no-duplicate-imports": "error",
         "no-else-return": "error",
         "no-eq-null": "error",
         "no-eval": "error",
@@ -94,18 +96,20 @@
         "no-undef-init": "error",
         "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
         "no-unused-expressions": "error",
-        "no-unused-vars": [
-            "error",
-            {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
-        ],
+        "no-unused-vars": ["error", {"ignoreRestSiblings": true}],
         "no-use-before-define": ["error", {"functions": false}],
         "no-useless-concat": "error",
         "no-useless-constructor": "error",
         "no-var": "error",
-        "object-shorthand": ["error", "always", {"avoidExplicitReturnArrows": true}],
+        "object-shorthand": "error",
         "one-var": ["error", "never"],
         "prefer-arrow-callback": "error",
-        "prefer-const": ["error", {"ignoreReadBeforeAssign": true}],
+        "prefer-const": [
+            "error",
+            {
+                "ignoreReadBeforeAssign": true
+            }
+        ],
         "radix": "error",
         "sort-imports": ["error", {"ignoreDeclarationSort": true}],
         "spaced-comment": ["error", "always", {"markers": ["/"]}],
@@ -114,13 +118,15 @@
         "unicorn/explicit-length-check": "off",
         "unicorn/filename-case": "off",
         "unicorn/no-await-expression-member": "off",
-        "unicorn/no-negated-condition": "off",
+        "unicorn/no-nested-ternary": "off",
         "unicorn/no-null": "off",
         "unicorn/no-process-exit": "off",
         "unicorn/no-useless-undefined": "off",
+        "unicorn/number-literal-case": "off",
         "unicorn/numeric-separators-style": "off",
         "unicorn/prefer-module": "off",
         "unicorn/prefer-node-protocol": "off",
+        "unicorn/prefer-spread": "off",
         "unicorn/prefer-ternary": "off",
         "unicorn/prefer-top-level-await": "off",
         "unicorn/prevent-abbreviations": "off",
@@ -130,19 +136,19 @@
     },
     "overrides": [
         {
-            "files": ["web/tests/**"],
+            "files": ["frontend_tests/node_tests/**", "frontend_tests/zjsunit/**"],
             "rules": {
                 "no-jquery/no-selector-prop": "off"
             }
         },
         {
-            "files": ["web/e2e-tests/**"],
+            "files": ["frontend_tests/puppeteer_lib/**", "frontend_tests/puppeteer_tests/**"],
             "globals": {
                 "zulip_test": false
             }
         },
         {
-            "files": ["web/src/**"],
+            "files": ["static/js/**"],
             "globals": {
                 "StripeCheckout": false
             }
@@ -150,9 +156,7 @@
         {
             "files": ["**/*.ts"],
             "extends": [
-                "plugin:@typescript-eslint/recommended",
                 "plugin:@typescript-eslint/recommended-requiring-type-checking",
-                "plugin:@typescript-eslint/strict",
                 "plugin:import/typescript"
             ],
             "parserOptions": {
@@ -170,34 +174,37 @@
             },
             "rules": {
                 // Disable base rule to avoid conflict
+                "no-duplicate-imports": "off",
+                "no-unused-vars": "off",
+                "no-useless-constructor": "off",
                 "no-use-before-define": "off",
 
-                "@typescript-eslint/consistent-type-assertions": [
-                    "error",
-                    {"assertionStyle": "never"}
-                ],
-                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
+                "@typescript-eslint/array-type": "error",
+                "@typescript-eslint/consistent-type-assertions": "error",
                 "@typescript-eslint/consistent-type-imports": "error",
                 "@typescript-eslint/explicit-function-return-type": [
                     "error",
                     {"allowExpressions": true}
                 ],
                 "@typescript-eslint/member-ordering": "error",
+                "@typescript-eslint/no-duplicate-imports": "off",
+                "@typescript-eslint/no-explicit-any": "off",
+                "@typescript-eslint/no-extraneous-class": "error",
                 "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-unnecessary-condition": "off",
+                "@typescript-eslint/no-parameter-properties": "error",
                 "@typescript-eslint/no-unnecessary-qualifier": "error",
+                "@typescript-eslint/no-unused-vars": ["error", {"ignoreRestSiblings": true}],
                 "@typescript-eslint/no-unsafe-argument": "off",
                 "@typescript-eslint/no-unsafe-assignment": "off",
                 "@typescript-eslint/no-unsafe-call": "off",
                 "@typescript-eslint/no-unsafe-member-access": "off",
                 "@typescript-eslint/no-unsafe-return": "off",
-                "@typescript-eslint/no-unused-vars": [
-                    "error",
-                    {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
-                ],
-                "@typescript-eslint/no-use-before-define": ["error", {"functions": false}],
-                "@typescript-eslint/parameter-properties": "error",
+                "@typescript-eslint/no-use-before-define": "error",
+                "@typescript-eslint/no-useless-constructor": "error",
+                "@typescript-eslint/prefer-includes": "error",
+                "@typescript-eslint/prefer-string-starts-ends-with": "error",
                 "@typescript-eslint/promise-function-async": "error",
+                "@typescript-eslint/unified-signatures": "error",
                 "no-undef": "error"
             }
         },
@@ -208,7 +215,7 @@
             }
         },
         {
-            "files": ["web/e2e-tests/**", "web/tests/**"],
+            "files": ["frontend_tests/**"],
             "globals": {
                 "CSS": false,
                 "document": false,
@@ -223,7 +230,7 @@
             }
         },
         {
-            "files": ["web/debug-require.js"],
+            "files": ["tools/debug-require.js"],
             "env": {
                 "browser": true,
                 "es2020": false
@@ -237,27 +244,20 @@
             }
         },
         {
-            "files": ["web/shared/**", "web/src/**", "web/third/**"],
+            "files": ["static/**"],
             "env": {
                 "browser": true,
                 "node": false
             },
-            "globals": {
-                "ZULIP_VERSION": false
-            },
             "rules": {
                 "no-console": "error"
             },
             "settings": {
-                "import/resolver": {
-                    "webpack": {
-                        "config": "./web/webpack.config.ts"
-                    }
-                }
+                "import/resolver": "webpack"
             }
         },
         {
-            "files": ["web/shared/**"],
+            "files": ["static/shared/**"],
             "env": {
                 "browser": false,
                 "shared-node-browser": true
@@ -268,14 +268,13 @@
                     {
                         "zones": [
                             {
-                                "target": "./web/shared",
+                                "target": "./static/shared",
                                 "from": ".",
-                                "except": ["./node_modules", "./web/shared"]
+                                "except": ["./node_modules", "./static/shared"]
                             }
                         ]
                     }
-                ],
-                "unicorn/prefer-string-replace-all": "off"
+                ]
             }
         }
     ]

.github/ISSUE_TEMPLATE/1_discussed_on_czo.md

@@ -1,10 +0,0 @@
----
-name: Issue discussed in the Zulip development community
-about: Bug report, feature or improvement already discussed on chat.zulip.org.
----
-
-<!-- Issue description -->
-
-<!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
-
-CZO thread

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -1,17 +0,0 @@
----
-name: Bug report
-about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
-labels: ["bug"]
----
-
-<!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
-
-<!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
-
-**Zulip Server and web app version:**
-
-- [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 7.0+
-- [ ] Zulip Server 6.0+
-- [ ] Zulip Server 5.0 or older
-- [ ] Other or not sure

.github/ISSUE_TEMPLATE/3_feature_request.md

@@ -1,6 +0,0 @@
----
-name: Feature or improvement request
-about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
----
-
-<!-- Describe the proposal, including how it would help you or your organization. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Possible bug
-    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
-    about: Report unexpected behavior that may be a bug.
-  - name: Feature suggestion or feedback
-    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
-    about: Start a discussion about your idea for improving Zulip.
-  - name: Issue with running or upgrading a Zulip server
-    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
-    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
-  - name: Other support requests and sales questions
-    url: https://zulip.com/help/contact-support
-    about: Contact us — we're happy to help!

.github/pull_request_template.md

@@ -9,8 +9,7 @@ Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gi
 
 **Screenshots and screen captures:**
 
-<details>
-<summary>Self-review checklist</summary>
+**Self-review checklist**
 
 <!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
 https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
@@ -28,7 +27,7 @@ Communicate decisions, questions, and potential concerns.
 - [ ] Calls out remaining decisions and concerns.
 - [ ] Automated tests verify logic where appropriate.
 
-Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
+Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html)).
 
 - [ ] Each commit is a coherent idea.
 - [ ] Commit message(s) explain reasoning and motivation for changes.
@@ -40,4 +39,3 @@ Completed manual review and testing of the following:
 - [ ] Strings and tooltips.
 - [ ] End-to-end functionality of buttons, interactions and flows.
 - [ ] Corner cases, error conditions, and easily imagined bugs.
-</details>

.github/workflows/production-suite.yml

@@ -8,16 +8,17 @@ on:
     paths:
       - .github/workflows/production-suite.yml
       - "**/migrations/**"
+      - babel.config.js
       - manage.py
-      - pnpm-lock.yaml
+      - postcss.config.js
       - puppet/**
       - requirements/**
       - scripts/**
+      - static/assets/**
+      - static/third/**
       - tools/**
-      - web/babel.config.js
-      - web/postcss.config.js
-      - web/third/**
-      - web/webpack.config.ts
+      - webpack.config.ts
+      - yarn.lock
       - zerver/worker/queue_processors.py
       - zerver/lib/push_notifications.py
       - zerver/decorator.py
@@ -46,7 +47,6 @@ jobs:
     # the top explain how to build and upload these images.
     # Ubuntu 20.04 ships with Python 3.8.10.
     container: zulip/ci:focal
-
     steps:
       - name: Add required permissions
         run: |
@@ -68,15 +68,16 @@ jobs:
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
+      - name: Restore node_modules cache
         uses: actions/cache@v3
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-focal-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-focal-${{ hashFiles('package.json') }}-${{ hashFiles('yarn.lock') }}
+          restore-keys: v1-yarn-deps-focal
 
       - name: Restore python cache
         uses: actions/cache@v3
@@ -100,30 +101,13 @@ jobs:
         with:
           name: production-tarball
           path: /tmp/production-build
-          retention-days: 1
+          retention-days: 14
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message
 
   production_install:
     # This job installs the server release tarball built above on a
@@ -132,29 +116,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        extra-args: [""]
         include:
           # Docker images are built from 'tools/ci/Dockerfile'; the comments at
           # the top explain how to build and upload these images.
           - docker_image: zulip/ci:focal
-            name: Ubuntu 20.04 production install and PostgreSQL upgrade with pgroonga
+            name: Ubuntu 20.04 production install
             os: focal
-            extra-args: ""
 
           - docker_image: zulip/ci:jammy
             name: Ubuntu 22.04 production install
             os: jammy
-            extra-args: ""
 
           - docker_image: zulip/ci:bullseye
             name: Debian 11 production install with custom db name and user
             os: bullseye
             extra-args: --test-custom-db
 
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install
-            os: bookworm
-            extra-args: ""
-
     name: ${{ matrix.name  }}
     container:
       image: ${{ matrix.docker_image }}
@@ -182,14 +160,21 @@ jobs:
           chmod +x /tmp/production-pgroonga
           chmod +x /tmp/production-install
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
+          chmod +x /tmp/send-failure-message
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
+      - name: Restore node_modules cache
+        uses: actions/cache@v3
+        with:
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('/tmp/package.json') }}-${{ hashFiles('/tmp/yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
+
       - name: Install production
         run: sudo /tmp/production-install ${{ matrix.extra-args }}
 
@@ -212,22 +197,11 @@ jobs:
         if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message
 
   production_upgrade:
     # The production upgrade job starts with a container with a
@@ -251,9 +225,6 @@ jobs:
           - docker_image: zulip/ci:bullseye-5.0
             name: 5.0 Version Upgrade
             os: bullseye
-          - docker_image: zulip/ci:bullseye-6.0
-            name: 6.0 Version Upgrade
-            os: bullseye
 
     name: ${{ matrix.name  }}
     container:
@@ -280,25 +251,14 @@ jobs:
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
+          chmod +x /tmp/send-failure-message
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Temporarily bootstrap PostgreSQL upgrades
-        # https://chat.zulip.org/#narrow/stream/43-automated-testing/topic/postgres.20client.20upgrade.20failures/near/1640444
-        # On Debian, there is an ordering issue with post-install maintainer
-        # scripts when postgresql-client-common is upgraded at the same time as
-        # postgresql-client and postgresql-client-15.  Upgrade just
-        # postgresql-client-common first, so the main upgrade process can
-        # succeed.  This is a _temporary_ work-around to improve CI signal, as
-        # the failure does represent a real failure that production systems may
-        # encounter.
-        run: sudo apt-get update && sudo apt-get install -y --only-upgrade postgresql-client-common
-
       - name: Upgrade production
         run: sudo /tmp/production-upgrade
 
@@ -308,19 +268,8 @@ jobs:
         # - name: Verify install
         #   run: sudo /tmp/production-verify
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message

.github/workflows/update-oneclick-apps.yml

@@ -22,6 +22,6 @@ jobs:
         run: |
           export PATH="$HOME/.local/bin:$PATH"
           git clone https://github.com/zulip/marketplace-partners
-          pip3 install python-digitalocean zulip fab-classic PyNaCl
+          pip3 install python-digitalocean zulip fab-classic
           echo $PATH
           python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

.github/workflows/zulip-ci.yml

@@ -27,6 +27,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        include_documentation_tests: [false]
+        include_frontend_tests: [false]
         include:
           # Base images are built using `tools/ci/Dockerfile.prod.template`.
           # The comments at the top explain how to build and upload these images.
@@ -34,26 +36,16 @@ jobs:
           - docker_image: zulip/ci:focal
             name: Ubuntu 20.04 (Python 3.8, backend + frontend)
             os: focal
-            include_documentation_tests: false
             include_frontend_tests: true
           # Debian 11 ships with Python 3.9.2.
           - docker_image: zulip/ci:bullseye
             name: Debian 11 (Python 3.9, backend + documentation)
             os: bullseye
             include_documentation_tests: true
-            include_frontend_tests: false
           # Ubuntu 22.04 ships with Python 3.10.4.
           - docker_image: zulip/ci:jammy
             name: Ubuntu 22.04 (Python 3.10, backend)
             os: jammy
-            include_documentation_tests: false
-            include_frontend_tests: false
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend)
-            os: bookworm
-            include_documentation_tests: false
-            include_frontend_tests: false
 
     runs-on: ubuntu-latest
     name: ${{ matrix.name }}
@@ -72,15 +64,16 @@ jobs:
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
+      - name: Restore node_modules cache
         uses: actions/cache@v3
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('package.json', 'yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
 
       - name: Restore python cache
         uses: actions/cache@v3
@@ -100,7 +93,10 @@ jobs:
         run: |
           # This is the main setup job for the test suite
           ./tools/ci/setup-backend --skip-dev-db-build
-          scripts/lib/clean_unused_caches.py --verbose --threshold=0
+
+          # Cleaning caches is mostly unnecessary in GitHub Actions, because
+          # most builds don't get to write to the cache.
+          # scripts/lib/clean_unused_caches.py --verbose --threshold 0
 
       - name: Run tools test
         run: |
@@ -112,26 +108,11 @@ jobs:
           source tools/ci/activate-venv
           ./tools/run-codespell
 
-      # We run the tests that are only run in a specific job early, so
-      # that we get feedback to the developer about likely failures as
-      # quickly as possible. Backend/mypy failures that aren't
-      # identical across different versions are much more rare than
-      # frontend linter or node test failures.
-      - name: Run documentation and api tests
-        if: ${{ matrix.include_documentation_tests }}
+      - name: Run backend lint
         run: |
           source tools/ci/activate-venv
-          # In CI, we only test links we control in test-documentation to avoid flakes
-          ./tools/test-documentation --skip-external-links
-          ./tools/test-help-documentation --skip-external-links
-          ./tools/test-api
-
-      - name: Run node tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Run the node tests first, since they're fast and deterministic
-          ./tools/test-js-with-node --coverage --parallel=1
+          echo "Test suite is running under $(python --version)."
+          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
 
       - name: Run frontend lint
         if: ${{ matrix.include_frontend_tests }}
@@ -139,41 +120,10 @@ jobs:
           source tools/ci/activate-venv
           ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
 
-      - name: Check schemas
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Check that various schemas are consistent. (is fast)
-          ./tools/check-schemas
-
-      - name: Check capitalization of strings
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./manage.py makemessages --locale en
-          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
-          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
-
-      - name: Run puppeteer tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-js-with-puppeteer
-
-      - name: Check pnpm dedupe
-        if: ${{ matrix.include_frontend_tests }}
-        run: pnpm dedupe --check
-
-      - name: Run backend lint
-        run: |
-          source tools/ci/activate-venv
-          echo "Test suite is running under $(python --version)."
-          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
-
       - name: Run backend tests
         run: |
           source tools/ci/activate-venv
-          ./tools/test-backend --coverage --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
+          ./tools/test-backend --coverage --include-webhooks --no-cov-cleanup --ban-console-output
 
       - name: Run mypy
         run: |
@@ -200,14 +150,50 @@ jobs:
           ./tools/test-migrations
           ./tools/setup/optimize-svg --check
           ./tools/setup/generate_integration_bots_avatars.py --check-missing
-          ./tools/ci/check-executables
 
-          # Ban check-database-compatibility from transitively
+          # Ban check-database-compatibility.py from transitively
           # relying on static/generated, because it might not be
           # up-to-date at that point in upgrade-zulip-stage-2.
-          chmod 000 static/generated web/generated
-          ./scripts/lib/check-database-compatibility
-          chmod 755 static/generated web/generated
+          chmod 000 static/generated
+          ./scripts/lib/check-database-compatibility.py
+          chmod 755 static/generated
+
+      - name: Run documentation and api tests
+        if: ${{ matrix.include_documentation_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # In CI, we only test links we control in test-documentation to avoid flakes
+          ./tools/test-documentation --skip-external-links
+          ./tools/test-help-documentation --skip-external-links
+          ./tools/test-api
+
+      - name: Run node tests
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # Run the node tests first, since they're fast and deterministic
+          ./tools/test-js-with-node --coverage --parallel=1
+
+      - name: Check schemas
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # Check that various schemas are consistent. (is fast)
+          ./tools/check-schemas
+
+      - name: Check capitalization of strings
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          ./manage.py makemessages --locale en
+          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
+          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
+
+      - name: Run puppeteer tests
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          ./tools/test-js-with-puppeteer
 
       - name: Check for untracked files
         run: |
@@ -247,25 +233,8 @@ jobs:
       - name: Check development database build
         run: ./tools/ci/setup-backend
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message

.gitignore

@@ -33,7 +33,6 @@ package-lock.json
 !/var/puppeteer/test_credentials.d.ts
 
 /.dmypy.json
-/.ruff_cache
 
 # Generated i18n data
 /locale/en
@@ -44,11 +43,11 @@ package-lock.json
 # Static build
 *.mo
 npm-debug.log
-/.pnpm-store
 /node_modules
 /prod-static
 /staticfiles.json
 /webpack-stats-production.json
+/yarn-error.log
 zulip-git-version
 
 # Test / analysis tools
@@ -83,9 +82,6 @@ zulip.kdev4
 # Core dump files
 core
 
-# Static generated files for landing page.
-/static/images/landing-page/hello/generated
-
 ## Miscellaneous
 # (Ideally this section is empty.)
 .transifexrc

.gitlint

@@ -1,13 +1,13 @@
 [general]
 ignore=title-trailing-punctuation, body-min-length, body-is-missing
 
-extra-path=tools/lib/gitlint_rules.py
+extra-path=tools/lib/gitlint-rules.py
 
 [title-match-regex]
 regex=^(.+:\ )?[A-Z].+\.$
 
 [title-max-length]
-line-length=72
+line-length=76
 
 [body-max-line-length]
 line-length=76

.mailmap

@@ -12,121 +12,72 @@
 #     # shows raw names/emails, filtered by mapped name:
 #   $ git log --format='%an %ae' --author=$NAME | uniq -c
 
-acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
-acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
-Adam Benesh <Adam.Benesh@gmail.com>
 Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
-Adarsh Tiwari <xoldyckk@gmail.com>
+Adam Benesh <Adam.Benesh@gmail.com>
 Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
 Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
 Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
-Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
-Aman Agrawal <amanagr@zulip.com>
 Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
 Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
 Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Aryan Shridhar <aryanshridhar7@gmail.com>
 Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
+Aryan Shridhar <aryanshridhar7@gmail.com>
 Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
 Austin Riba <austin@zulip.com> <austin@m51.io>
 BIKI DAS <bikid475@gmail.com>
-Brijmohan Siyag <brijsiyag@gmail.com>
 Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
-Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
+Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
 Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dinesh <chdinesh1089@gmail.com>
-Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
 Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
-Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
-Evy Kassirer <evy@zulip.com>
-Evy Kassirer <evy@zulip.com> <evy.kassirer@gmail.com>
-Evy Kassirer <evy@zulip.com> <evykassirer@users.noreply.github.com>
-Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
 Greg Price <greg@zulip.com> <gnprice@gmail.com>
 Greg Price <greg@zulip.com> <greg@zulipchat.com>
 Greg Price <greg@zulip.com> <price@mit.edu>
-Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
-Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
 Jai soni <jai_s@me.iitr.ac.in>
 Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-Joseph Ho <josephho678@gmail.com>
-Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
 Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
-Karl Stolley <karl@zulip.com> <karl@stolley.dev>
 Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
 Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
 Kevin Scott <kevin.scott.98@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
 Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
 Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
-Matt Keller <matt@zulip.com>
-Matt Keller <matt@zulip.com> <m@cognusion.com>
-Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
-nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
-Palash Baderia <palash.baderia@outlook.com>
-Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
+m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
 Palash Raghuwanshi <singhpalash0@gmail.com>
 Parth <mittalparth22@gmail.com>
 Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
 Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
 Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
-Rein Zustand (rht) <rhtbot@protonmail.com>
-Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
 Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
 Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
-Rixant Rokaha <rixantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
-Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Satyam Bansal <sbansal1999@gmail.com>
+Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Sayam Samal <samal.sayam@gmail.com>
 Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
 Shlok Patel <shlokcpatel2001@gmail.com>
-Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
 Steve Howell <showell@zulip.com> <showell30@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@zulipchat.com>
 Steve Howell <showell@zulip.com> <steve@humbughq.com>
 Steve Howell <showell@zulip.com> <steve@zulip.com>
 strifel <info@strifel.de>
-Tim Abbott <tabbott@zulip.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
 Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
-umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
-umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
 Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
 Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com>
+Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
+Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
 Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
+Yash RE <33805964+YashRE42@users.noreply.github.com>
 Yogesh Sirsat <yogeshsirsat56@gmail.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com>
 Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
-Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
-Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
-Zev Benjamin <zev@zulip.com> <zev@mit.edu>
-Zixuan James Li <p359101898@gmail.com>
-Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
-Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>
+Zeeshan Equbal <equbalzeeshan@gmail.com>

.npmrc

@@ -1 +0,0 @@
-ignore-dep-scripts=true

.prettierignore

@@ -1,11 +1,8 @@
-pnpm-lock.yaml
-/api_docs/**/*.md
 /corporate/tests/stripe_fixtures
-/help/**/*.md
 /locale
+/static/third
 /templates/**/*.md
 /tools/setup/emoji/emoji_map.json
-/web/third
 /zerver/tests/fixtures
 /zerver/webhooks/*/doc.md
 /zerver/webhooks/*/fixtures

.readthedocs.yaml

@@ -1,15 +0,0 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
-
-python:
-  install:
-    - requirements: requirements/docs.txt

.tx/config

@@ -1,39 +1,32 @@
-# Migrated from transifex-client format with `tx migrate`
-#
-# See https://developers.transifex.com/docs/using-the-client which hints at
-# this format, but in general, the headings are in the format of:
-#
-# [o:<org>:p:<project>:r:<resource>]
-
 [main]
 host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
-[o:zulip:p:zulip:r:djangopo]
+[zulip.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip:r:mobile]
+[zulip.translationsjson]
+file_filter = locale/<lang>/translations.json
+source_file = locale/en/translations.json
+source_lang = en
+type = KEYVALUEJSON
+
+[zulip.mobile]
 file_filter = locale/<lang>/mobile.json
 source_file = locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 
-[o:zulip:p:zulip:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip-test:r:djangopo]
+[zulip-test.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip-test:r:translationsjson]
+[zulip-test.translationsjson]
 file_filter = locale/<lang>/translations.json
 source_file = locale/en/translations.json
 source_lang = en

.yarnrc

@@ -0,0 +1 @@
+ignore-scripts true

CODE_OF_CONDUCT.md

@@ -102,72 +102,3 @@ This Code of Conduct is adapted from the
 under a
 [Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
 license.
-
-## Moderating the Zulip community
-
-Anyone can help moderate the Zulip community by helping make sure that folks are
-aware of the [community guidelines](https://zulip.com/development-community/)
-and this Code of Conduct, and that we maintain a positive and respectful
-atmosphere.
-
-Here are some guidelines for you how can help:
-
-- Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
-  and just trying to keep the atmosphere warm make the whole community function
-  more smoothly. New participants who feel accepted, listened to and respected
-  are likely to treat others the same way.
-
-- Be familiar with the [community
-  guidelines](https://zulip.com/development-community/), and cite them liberally
-  when a user violates them. Be polite but firm. Some examples:
-
-  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
-    you ask a question. As noted in the [guidelines for this
-    community](https://zulip.com/development-community/):
-
-    > Use @-mentions sparingly… there is generally no need to @-mention a
-    > core contributor unless you need their timely attention.
-
-  - @user, please keep in mind the following [community
-    guideline](https://zulip.com/development-community/):
-
-    > Don’t ask the same question in multiple places. Moderators read every
-    > public stream, and make sure every question gets a reply.
-
-    I’ve gone ahead and moved the other copy of this message to this thread.
-
-  - If asked a question in a direct message that is better discussed in a public
-    stream:
-    > Hi @user! Please start by reviewing
-    > https://zulip.com/development-community/#community-norms to learn how to
-    > get help in this community.
-
-- Users sometimes think chat.zulip.org is a testing instance. When this happens,
-  kindly direct them to use the **#test here** stream.
-
-- If you see a message that’s posted in the wrong place, go ahead and move it if
-  you have permissions to do so, even if you don’t plan to respond to it.
-  Leaving the “Send automated notice to new topic” option enabled helps make it
-  clear what happened to the person who sent the message.
-
-  If you are responding to a message that's been moved, mention the user in your
-  reply, so that the mention serves as a notification of the new location for
-  their conversation.
-
-- If a user is posting spam, please report it to an administrator. They will:
-
-  - Change the user's name to `<name> (spammer)` and deactivate them.
-  - Delete any spam messages they posted in public streams.
-
-- We care very much about maintaining a respectful tone in our community. If you
-  see someone being mean or rude, point out that their tone is inappropriate,
-  and ask them to communicate their perspective in a respectful way in the
-  future. If you don’t feel comfortable doing so yourself, feel free to ask a
-  member of Zulip's core team to take care of the situation.
-
-- Try to assume the best intentions from others (given the range of
-  possibilities presented by their visible behavior), and stick with a friendly
-  and positive tone even when someone’s behavior is poor or disrespectful.
-  Everyone has bad days and stressful situations that can result in them
-  behaving not their best, and while we should be firm about our community
-  rules, we should also enforce them with kindness.

CONTRIBUTING.md

@@ -55,10 +55,8 @@ needs doing:
 **Non-code contributions**: Some of the most valuable ways to contribute
 don't require touching the codebase at all. For example, you can:
 
-- Report issues, including both [feature
-  requests](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html)
-  and [bug
-  reports](https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html).
+- [Report issues](#reporting-issues), including both feature requests and
+  bug reports.
 - [Give feedback](#user-feedback) if you are evaluating or using Zulip.
 - [Participate
   thoughtfully](https://zulip.readthedocs.io/en/latest/contributing/design-discussions.html)
@@ -78,13 +76,12 @@ to help.
 
 - First, make an account on the
   [Zulip community server](https://zulip.com/development-community/),
-  paying special attention to the
-  [community norms](https://zulip.com/development-community/#community-norms).
-  If you'd like, introduce yourself in
+  paying special attention to the community norms. If you'd like, introduce
+  yourself in
   [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
   your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken or interesting/helpful as you
-  started using the product.
+  Zulip, and anything that felt confusing/broken as you started using the
+  product.
 - Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
 - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
   getting help in
@@ -149,15 +146,19 @@ Note that you are _not_ claiming an issue while you are iterating through steps
 1-4. _Before you claim an issue_, you should be confident that you will be able to
 tackle it effectively.
 
+If the lists of issues are overwhelming, you can post in
+[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
+bit about your background and interests, and we'll help you out. The most
+important thing to say is whether you're looking for a backend (Python),
+frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
+documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
+bit about your programming experience and available time.
+
 Additional tips for the [main server and web app
 repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
 
 - We especially recommend browsing recently opened issues, as there are more
   likely to be easy ones for you to find.
-- Take a look at issues with the ["good first issue"
-  label](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22),
-  as they are especially accessible to new contributors. However, you will
-  likely find issues without this label that are accessible as well.
 - All issues are partitioned into areas like
   admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
   through our [list of labels](https://github.com/zulip/zulip/labels), and
@@ -214,16 +215,101 @@ stream](https://chat.zulip.org/#narrow/stream/101-design) in the [Zulip
 development community](https://zulip.com/development-community/)
 
 For more advice, see [What makes a great Zulip
-contributor?](#what-makes-a-great-zulip-contributor) below. It's OK if your
-first issue takes you a while; that's normal! You'll be able to work a lot
-faster as you build experience.
+contributor?](#what-makes-a-great-zulip-contributor)
+below.
 
 ### Submitting a pull request
 
-See the [pull request review
-process](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-guide for detailed instructions on how to submit a pull request, and information
-on the stages of review your PR will go through.
+When you believe your code is ready, follow the [guide on how to review
+code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
+to review your own work. You can often find things you missed by taking a step
+back to look over your work before asking others to do so. Catching mistakes
+yourself will help your PRs be merged faster, and folks will appreciate the
+quality and professionalism of your work.
+
+Then, submit your changes. Carefully reading our [Git guide][git-guide], and in
+particular the section on [making a pull request][git-guide-make-pr], will help
+avoid many common mistakes. If any part of your contribution is from someone
+else (code snippets, images, sounds, or any other copyrightable work, modified
+or unmodified), be sure to review the instructions on how to [properly
+attribute][licensing] the work.
+
+[licensing]: https://zulip.readthedocs.io/en/latest/contributing/licensing.html#contributing-someone-else-s-work
+
+Once you are satisfied with the quality of your PR, follow the
+[guidelines on asking for a code
+review](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#asking-for-a-code-review)
+to request a review. If you are not sure what's best, simply post a
+comment on the main GitHub thread for your PR clearly indicating that
+it is ready for review, and the project maintainers will take a look
+and follow up with next steps.
+
+It's OK if your first issue takes you a while; that's normal! You'll be
+able to work a lot faster as you build experience.
+
+If it helps your workflow, you can submit your pull request marked as
+a [draft][github-help-draft-pr] while you're still working on it, and
+then mark it ready when you think it's time for someone else to review
+your work.
+
+[git-guide]: https://zulip.readthedocs.io/en/latest/git/
+[git-guide-make-pr]: https://zulip.readthedocs.io/en/latest/git/pull-requests.html
+[github-help-draft-pr]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests
+
+### Stages of a pull request
+
+Your pull request will likely go through several stages of review.
+
+1. If your PR makes user-facing changes, the UI and user experience may be
+   reviewed early on, without reference to the code. You will get feedback on
+   any user-facing bugs in the implementation. To minimize the number of review
+   round-trips, make sure to [thoroughly
+   test](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#manual-testing)
+   your own PR prior to asking for review.
+2. There may be choices made in the implementation that the reviewer
+   will ask you to revisit. This process will go more smoothly if you
+   specifically call attention to the decisions you made while
+   drafting the PR and any points about which you are uncertain. The
+   PR description and comments on your own PR are good ways to do this.
+3. Oftentimes, seeing an initial implementation will make it clear that the
+   product design for a feature needs to be revised, or that additional changes
+   are needed. The reviewer may therefore ask you to amend or change the
+   implementation. Some changes may be blockers for getting the PR merged, while
+   others may be improvements that can happen afterwards. Feel free to ask if
+   it's unclear which type of feedback you're getting. (Follow-ups can be a
+   great next issue to work on!)
+4. In addition to any UI/user experience review, all PRs will go through one or
+   more rounds of code review. Your code may initially be [reviewed by other
+   contributors](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html).
+   This helps us make good use of project maintainers' time, and helps you make
+   progress on the PR by getting more frequent feedback. A project maintainer
+   may leave a comment asking someone with expertise in the area you're working
+   on to review your work.
+5. Final code review and integration for server and web app PRs is generally done
+   by `@timabbott`.
+
+#### How to help move the review process forward
+
+The key to keeping your review moving through the review process is to:
+
+- Address _all_ the feedback to the best of your ability.
+- Make it clear when the requested changes have been made
+  and you believe it's time for another look.
+- Make it as easy as possible to review the changes you made.
+
+In order to do this, when you believe you have addressed the previous round of
+feedback on your PR as best you can, post a comment asking reviewers to take
+another look. Your comment should make it easy to understand what has been done
+and what remains by:
+
+- Summarizing the changes made since the last review you received.
+- Highlighting remaining questions or decisions, with links to any relevant
+  chat.zulip.org threads.
+- Providing updated screenshots and information on manual testing if
+  appropriate.
+
+The easier it is to review your work, the more likely you are to receive quick
+feedback.
 
 ### Beyond the first issue
 
@@ -249,34 +335,17 @@ labels.
   use the existing pull request (PR) as a starting point for your contribution. If
   you think a different approach is needed, you can post a new PR, with a comment that clearly
   explains _why_ you decided to start from scratch.
-- **What if I ask if someone is still working on an issue, and they don't
-  respond?** If you don't get a reply within 2-3 days, go ahead and post a comment
-  that you are working on the issue, and submit a pull request. If the original
-  assignee ends up submitting a pull request first, no worries! You can help by
-  providing feedback on their work, or submit your own PR if you think a
-  different approach is needed (as described above).
 - **Can I come up with my own feature idea and work on it?** We welcome
   suggestions of features or other improvements that you feel would be valuable. If you
   have a new feature you'd like to add, you can start a conversation [in our
   development community](https://zulip.com/development-community/#where-do-i-send-my-message)
   explaining the feature idea and the problem that you're hoping to solve.
-- **I'm waiting for the next round of review on my PR. Can I pick up
-  another issue in the meantime?** Someone's first Zulip PR often
-  requires quite a bit of iteration, so please [make sure your pull
-  request is reviewable][reviewable-pull-requests] and go through at
-  least one round of feedback from others before picking up a second
-  issue. After that, sure! If
-  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
-  claim an issue, you can post a comment describing the status of your
-  other work on the issue you're interested in, and asking for the
-  issue to be assigned to you. Note that addressing feedback on
-  in-progress PRs should always take priority over starting a new PR.
 - **I think my PR is done, but it hasn't been merged yet. What's going on?**
   1. **Double-check that you have addressed all the feedback**, including any comments
      on [Git commit
-     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
+     discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
   2. If all the feedback has been addressed, did you [leave a
-     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
+     comment](#how-to-help-move-the-review-process-forward)
      explaining that you have done so and **requesting another review**? If not,
      it may not be clear to project maintainers or reviewers that your PR is
      ready for another look.
@@ -292,8 +361,6 @@ labels.
      occasionally take a few weeks for a PR in the final stages of the review
      process to be merged.
 
-[reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
-
 ## What makes a great Zulip contributor?
 
 Zulip has a lot of experience working with new contributors. In our
@@ -305,7 +372,7 @@ experience, these are the best predictors of success:
   you got stuck. Post tracebacks or other error messages if appropriate. For
   more advice, check out [our guide][great-questions]!
 - Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
+  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
 - Submitting carefully tested code. See our [detailed guide on how to review
   code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
   (yours or someone else's).
@@ -326,6 +393,29 @@ experience, these are the best predictors of success:
 
 [great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
 
+## Reporting issues
+
+If you find an easily reproducible bug and/or are experienced in reporting
+bugs, feel free to just open an issue on the relevant project on GitHub.
+
+If you have a feature request or are not yet sure what the underlying bug
+is, the best place to post issues is
+[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
+[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
+[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
+[Zulip community server](https://zulip.com/development-community/).
+This allows us to interactively figure out what is going on, let you know if
+a similar issue has already been opened, and collect any other information
+we need. Choose a 2-4 word topic that describes the issue, explain the issue
+and how to reproduce it if known, your browser/OS if relevant, and a
+[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
+if appropriate.
+
+**Reporting security issues**. Please do not report security issues
+publicly, including on public streams on chat.zulip.org. You can
+email [security@zulip.com](mailto:security@zulip.com). We create a CVE for every
+security issue in our released software.
+
 ## User feedback
 
 Nearly every feature we develop starts with a user request. If you are part

Dockerfile-postgresql

@@ -8,7 +8,7 @@
 #       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
 
 # Currently the PostgreSQL images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image cannot currently be upgraded
+# the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
 
 # https://hub.docker.com/r/groonga/pgroonga/tags

README.md

@@ -17,7 +17,6 @@ Come find us on the [development community chat](https://zulip.com/development-c
 [![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
 [![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)

SECURITY.md

@@ -33,5 +33,5 @@ See also our documentation on the [Zulip release
 lifecycle][release-lifecycle].
 
 [security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
-[upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
+[upgrades]: https://zulip.readthedocs.io/en/latest/production/upgrade-or-modify.html#upgrading-to-a-release
 [release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html

analytics/lib/counts.py

@@ -8,7 +8,6 @@ from django.conf import settings
 from django.db import connection, models
 from django.db.models import F
 from psycopg2.sql import SQL, Composable, Identifier, Literal
-from typing_extensions import TypeAlias, override
 
 from analytics.models import (
     BaseCount,
@@ -63,8 +62,7 @@ class CountStat:
         else:
             self.interval = self.time_increment
 
-    @override
-    def __repr__(self) -> str:
+    def __str__(self) -> str:
         return f"<CountStat: {self.property}>"
 
     def last_successful_fill(self) -> Optional[datetime]:
@@ -290,7 +288,6 @@ def do_aggregate_to_summary_table(
 
 ## Utility functions called from outside counts.py ##
 
-
 # called from zerver.actions; should not throw any errors
 def do_increment_logging_stat(
     zerver_object: Union[Realm, UserProfile, Stream],
@@ -318,7 +315,7 @@ def do_increment_logging_stat(
     else:  # CountStat.HOUR:
         end_time = ceiling_to_hour(event_time)
 
-    row, created = table._default_manager.get_or_create(
+    row, created = table.objects.get_or_create(
         property=stat.property,
         subgroup=subgroup,
         end_time=end_time,
@@ -348,7 +345,7 @@ def do_drop_single_stat(property: str) -> None:
 
 ## DataCollector-level operations ##
 
-QueryFn: TypeAlias = Callable[[Dict[str, Composable]], Composable]
+QueryFn = Callable[[Dict[str, Composable]], Composable]
 
 
 def do_pull_by_sql_query(
@@ -448,13 +445,7 @@ def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -481,13 +472,7 @@ def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -536,10 +521,7 @@ def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        realm_clause = SQL(
-            "zerver_stream.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_streamcount

analytics/lib/fixtures.py

@@ -1,5 +1,5 @@
 from math import sqrt
-from random import Random
+from random import gauss, random, seed
 from typing import List
 
 from analytics.lib.counts import CountStat
@@ -36,8 +36,6 @@ def generate_time_series_data(
     partial_sum -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
-    rng = Random(random_seed)
-
     if frequency == CountStat.HOUR:
         length = days * 24
         seasonality = [non_business_hours_base] * 24 * 7
@@ -46,13 +44,13 @@ def generate_time_series_data(
                 seasonality[24 * day + hour] = business_hours_base
         holidays = []
         for i in range(days):
-            holidays.extend([rng.random() < holiday_rate] * 24)
+            holidays.extend([random() < holiday_rate] * 24)
     elif frequency == CountStat.DAY:
         length = days
         seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
             24 * non_business_hours_base
         ] * 2
-        holidays = [rng.random() < holiday_rate for i in range(days)]
+        holidays = [random() < holiday_rate for i in range(days)]
     else:
         raise AssertionError(f"Unknown frequency: {frequency}")
     if length < 2:
@@ -64,10 +62,11 @@ def generate_time_series_data(
         seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)
     ]
 
-    noise_scalars = [rng.gauss(0, 1)]
+    seed(random_seed)
+    noise_scalars = [gauss(0, 1)]
     for i in range(1, length):
         noise_scalars.append(
-            noise_scalars[-1] * autocorrelation + rng.gauss(0, 1) * (1 - autocorrelation)
+            noise_scalars[-1] * autocorrelation + gauss(0, 1) * (1 - autocorrelation)
         )
 
     values = [

analytics/lib/time_utils.py

@@ -30,5 +30,4 @@ def time_range(
     while current >= start:
         times.append(current)
         current -= step
-    times.reverse()
-    return times
+    return list(reversed(times))

analytics/management/commands/check_analytics_state.py

@@ -5,11 +5,10 @@ from typing import Any, Dict
 
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.models import installation_epoch
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
+from zerver.lib.timestamp import TimeZoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC
 from zerver.models import Realm
 
 states = {
@@ -25,7 +24,6 @@ class Command(BaseCommand):
 
     Run as a cron job that runs every hour."""
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         fill_state = self.get_fill_state()
         status = fill_state["status"]
@@ -50,7 +48,7 @@ class Command(BaseCommand):
                 last_fill = installation_epoch()
             try:
                 verify_UTC(last_fill)
-            except TimeZoneNotUTCError:
+            except TimeZoneNotUTCException:
                 return {"status": 2, "message": f"FillState not in UTC for {property}"}
 
             if stat.frequency == CountStat.DAY:

analytics/management/commands/clear_analytics_tables.py

@@ -2,7 +2,6 @@ from argparse import ArgumentParser
 from typing import Any
 
 from django.core.management.base import BaseCommand, CommandError
-from typing_extensions import override
 
 from analytics.lib.counts import do_drop_all_analytics_tables
 
@@ -10,11 +9,9 @@ from analytics.lib.counts import do_drop_all_analytics_tables
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         if options["force"]:
             do_drop_all_analytics_tables()

analytics/management/commands/clear_single_stat.py

@@ -2,7 +2,6 @@ from argparse import ArgumentParser
 from typing import Any
 
 from django.core.management.base import BaseCommand, CommandError
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 
@@ -10,12 +9,10 @@ from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Actually do it.")
         parser.add_argument("--property", help="The property of the stat to be cleared.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         property = options["property"]
         if property not in COUNT_STATS:

analytics/management/commands/populate_analytics_db.py

@@ -5,7 +5,6 @@ from typing import Any, Dict, List, Mapping, Type, Union
 from django.core.files.uploadedfile import UploadedFile
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import TypeAlias, override
 
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
@@ -24,17 +23,8 @@ from zerver.lib.create_user import create_user
 from zerver.lib.storage import static_path
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
-from zerver.lib.upload import upload_message_attachment_from_request
-from zerver.models import (
-    Client,
-    Realm,
-    RealmAuditLog,
-    Recipient,
-    Stream,
-    Subscription,
-    UserGroup,
-    UserProfile,
-)
+from zerver.lib.upload import upload_message_image_from_request
+from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserGroup, UserProfile
 
 
 class Command(BaseCommand):
@@ -68,7 +58,6 @@ class Command(BaseCommand):
             random_seed=self.random_seed,
         )
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         # TODO: This should arguably only delete the objects
         # associated with the "analytics" realm.
@@ -127,29 +116,25 @@ class Command(BaseCommand):
         stream.save(update_fields=["recipient"])
 
         # Subscribe shylock to the stream to avoid invariant failures.
-        Subscription.objects.create(
-            recipient=recipient,
-            user_profile=shylock,
-            is_user_active=shylock.is_active,
-            color=STREAM_ASSIGNMENT_COLORS[0],
-        )
-        RealmAuditLog.objects.create(
-            realm=realm,
-            modified_user=shylock,
-            modified_stream=stream,
-            event_last_message_id=0,
-            event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
-            event_time=installation_time,
-        )
+        # TODO: This should use subscribe_users_to_streams from populate_db.
+        subs = [
+            Subscription(
+                recipient=recipient,
+                user_profile=shylock,
+                is_user_active=shylock.is_active,
+                color=STREAM_ASSIGNMENT_COLORS[0],
+            ),
+        ]
+        Subscription.objects.bulk_create(subs)
 
         # Create an attachment in the database for set_storage_space_used_statistic.
         IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
         file_info = os.stat(IMAGE_FILE_PATH)
         file_size = file_info.st_size
         with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock, file_size)
+            upload_message_image_from_request(UploadedFile(fp), shylock, file_size)
 
-        FixtureData: TypeAlias = Mapping[Union[str, int, None], List[int]]
+        FixtureData = Mapping[Union[str, int, None], List[int]]
 
         def insert_fixture_data(
             stat: CountStat,
@@ -157,7 +142,7 @@ class Command(BaseCommand):
             table: Type[BaseCount],
         ) -> None:
             end_times = time_range(
-                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
+                last_end_time, last_end_time, stat.frequency, len(list(fixture_data.values())[0])
             )
             if table == InstallationCount:
                 id_args: Dict[str, Any] = {}
@@ -169,7 +154,7 @@ class Command(BaseCommand):
                 id_args = {"stream": stream, "realm": realm}
 
             for subgroup, values in fixture_data.items():
-                table._default_manager.bulk_create(
+                table.objects.bulk_create(
                     table(
                         property=stat.property,
                         subgroup=subgroup,

analytics/management/commands/update_analytics_counts.py

@@ -8,11 +8,10 @@ from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
 from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.remote_server import send_analytics_to_push_bouncer
+from zerver.lib.remote_server import send_analytics_to_remote_server
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 
@@ -22,7 +21,6 @@ class Command(BaseCommand):
 
     Run as a cron job that runs every hour."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument(
             "--time",
@@ -39,7 +37,6 @@ class Command(BaseCommand):
             "--verbose", action="store_true", help="Print timing information to stdout."
         )
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         try:
             os.mkdir(settings.ANALYTICS_LOCK_DIR)
@@ -96,4 +93,4 @@ class Command(BaseCommand):
         logger.info("Finished updating analytics counts through %s", fill_to_time)
 
         if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
-            send_analytics_to_push_bouncer()
+            send_analytics_to_remote_server()

analytics/migrations/0001_initial.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0030_realm_org_type"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),

analytics/migrations/0002_remove_huddlecount.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0001_initial"),
     ]

analytics/migrations/0003_fillstate.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0002_remove_huddlecount"),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0003_fillstate"),
     ]

analytics/migrations/0005_alter_field_size.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0004_add_subgroup"),
     ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0005_alter_field_size"),
     ]

analytics/migrations/0007_remove_interval.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0006_add_subgroup_to_unique_constraints"),
     ]

analytics/migrations/0008_add_count_indexes.py

@@ -1,33 +1,25 @@
 # Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0050_userprofile_avatar_version"),
         ("analytics", "0007_remove_interval"),
     ]
 
     operations = [
-        migrations.AddIndex(
-            model_name="realmcount",
-            index=models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="realmcount",
+            index_together={("property", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="streamcount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="streamcount",
+            index_together={("property", "realm", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="usercount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="usercount",
+            index_together={("property", "realm", "end_time")},
         ),
     ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -21,6 +21,7 @@ def delete_messages_sent_to_stream_stat(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0008_add_count_indexes"),
     ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -21,6 +21,7 @@ def clear_message_sent_by_message_type_values(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
 
     operations = [

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -18,6 +18,7 @@ def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0010_clear_messages_sent_values"),
     ]

analytics/migrations/0012_add_on_delete.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0011_clear_analytics_tables"),
     ]

analytics/migrations/0013_remove_anomaly.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0012_add_on_delete"),
     ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0013_remove_anomaly"),
     ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 from django.db.models import Count, Sum
 
@@ -55,6 +55,7 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0014_remove_fillstate_last_modified"),
     ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0015_clear_duplicate_counts"),
     ]

analytics/models.py

@@ -1,11 +1,7 @@
-# https://github.com/typeddjango/django-stubs/issues/1698
-# mypy: disable-error-code="explicit-override"
-
 import datetime
 
 from django.db import models
 from django.db.models import Q, UniqueConstraint
-from typing_extensions import override
 
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Stream, UserProfile
@@ -20,9 +16,8 @@ class FillState(models.Model):
     STARTED = 2
     state = models.PositiveSmallIntegerField()
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.end_time} {self.state}"
+        return f"<FillState: {self.property} {self.end_time} {self.state}>"
 
 
 # The earliest/starting end_time in FillState
@@ -63,9 +58,8 @@ class InstallationCount(BaseCount):
             ),
         ]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.subgroup} {self.value}"
+        return f"<InstallationCount: {self.property} {self.subgroup} {self.value}>"
 
 
 class RealmCount(BaseCount):
@@ -85,16 +79,10 @@ class RealmCount(BaseCount):
                 name="unique_realm_count_null_subgroup",
             ),
         ]
-        indexes = [
-            models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            )
-        ]
+        index_together = ["property", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
+        return f"<RealmCount: {self.realm} {self.property} {self.subgroup} {self.value}>"
 
 
 class UserCount(BaseCount):
@@ -117,16 +105,10 @@ class UserCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from users to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
+        return f"<UserCount: {self.user} {self.property} {self.subgroup} {self.value}>"
 
 
 class StreamCount(BaseCount):
@@ -149,13 +131,9 @@ class StreamCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from streams to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"
+        return (
+            f"<StreamCount: {self.stream} {self.property} {self.subgroup} {self.value} {self.id}>"
+        )

analytics/tests/test_activity_views.py

@@ -3,7 +3,7 @@ from unittest import mock
 from django.utils.timezone import now as timezone_now
 
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.models import Client, UserActivity, UserProfile
+from zerver.models import Client, UserActivity, UserProfile, flush_per_request_caches
 
 
 class ActivityTest(ZulipTestCase):
@@ -31,23 +31,18 @@ class ActivityTest(ZulipTestCase):
         user_profile.is_staff = True
         user_profile.save(update_fields=["is_staff"])
 
-        with self.assert_database_query_count(12):
+        flush_per_request_caches()
+        with self.assert_database_query_count(19):
             result = self.client_get("/activity")
             self.assertEqual(result.status_code, 200)
 
-        with self.assert_database_query_count(4):
-            result = self.client_get("/activity/remote")
-            self.assertEqual(result.status_code, 200)
-
-        with self.assert_database_query_count(4):
-            result = self.client_get("/activity/integrations")
-            self.assertEqual(result.status_code, 200)
-
+        flush_per_request_caches()
         with self.assert_database_query_count(8):
             result = self.client_get("/realm_activity/zulip/")
             self.assertEqual(result.status_code, 200)
 
         iago = self.example_user("iago")
+        flush_per_request_caches()
         with self.assert_database_query_count(5):
             result = self.client_get(f"/user_activity/{iago.id}/")
             self.assertEqual(result.status_code, 200)

analytics/tests/test_counts.py

@@ -8,7 +8,6 @@ from django.db import models
 from django.db.models import Sum
 from django.utils.timezone import now as timezone_now
 from psycopg2.sql import SQL, Literal
-from typing_extensions import override
 
 from analytics.lib.counts import (
     COUNT_STATS,
@@ -54,7 +53,7 @@ from zerver.actions.users import do_deactivate_user
 from zerver.lib.create_user import create_user
 from zerver.lib.exceptions import InvitationError
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day
+from zerver.lib.timestamp import TimeZoneNotUTCException, floor_to_day
 from zerver.lib.topic import DB_TOPIC_NAME
 from zerver.lib.utils import assert_is_not_none
 from zerver.models import (
@@ -82,7 +81,6 @@ class AnalyticsTestCase(ZulipTestCase):
     TIME_ZERO = datetime(1988, 3, 14, tzinfo=timezone.utc)
     TIME_LAST_HOUR = TIME_ZERO - HOUR
 
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.default_realm = do_create_realm(
@@ -184,9 +182,7 @@ class AnalyticsTestCase(ZulipTestCase):
     ) -> None:
         if property is None:
             property = self.current_property
-        queryset = table._default_manager.filter(property=property, end_time=end_time).filter(
-            **kwargs
-        )
+        queryset = table.objects.filter(property=property, end_time=end_time).filter(**kwargs)
         if table is not InstallationCount:
             if realm is None:
                 realm = self.default_realm
@@ -231,15 +227,16 @@ class AnalyticsTestCase(ZulipTestCase):
                 kwargs[arg_keys[i]] = values[i]
             for key, value in defaults.items():
                 kwargs[key] = kwargs.get(key, value)
-            if table is not InstallationCount and "realm" not in kwargs:
-                if "user" in kwargs:
-                    kwargs["realm"] = kwargs["user"].realm
-                elif "stream" in kwargs:
-                    kwargs["realm"] = kwargs["stream"].realm
-                else:
-                    kwargs["realm"] = self.default_realm
-            self.assertEqual(table._default_manager.filter(**kwargs).count(), 1)
-        self.assert_length(arg_values, table._default_manager.count())
+            if table is not InstallationCount:
+                if "realm" not in kwargs:
+                    if "user" in kwargs:
+                        kwargs["realm"] = kwargs["user"].realm
+                    elif "stream" in kwargs:
+                        kwargs["realm"] = kwargs["stream"].realm
+                    else:
+                        kwargs["realm"] = self.default_realm
+            self.assertEqual(table.objects.filter(**kwargs).count(), 1)
+        self.assert_length(arg_values, table.objects.count())
 
 
 class TestProcessCountStat(AnalyticsTestCase):
@@ -293,7 +290,7 @@ class TestProcessCountStat(AnalyticsTestCase):
         stat = self.make_dummy_count_stat("test stat")
         with self.assertRaises(ValueError):
             process_count_stat(stat, installation_epoch() + 65 * self.MINUTE)
-        with self.assertRaises(TimeZoneNotUTCError):
+        with self.assertRaises(TimeZoneNotUTCException):
             process_count_stat(stat, installation_epoch().replace(tzinfo=None))
 
     # This tests the LoggingCountStat branch of the code in do_delete_counts_at_hour.
@@ -457,7 +454,6 @@ class TestProcessCountStat(AnalyticsTestCase):
 
 
 class TestCountStats(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         # This tests two things for each of the queries/CountStats: Handling
@@ -662,7 +658,7 @@ class TestCountStats(AnalyticsTestCase):
         self.create_message(user1, recipient_huddle1)
         self.create_message(user2, recipient_huddle2)
 
-        # direct messages
+        # private messages
         recipient_user1 = Recipient.objects.get(type_id=user1.id, type=Recipient.PERSONAL)
         recipient_user2 = Recipient.objects.get(type_id=user2.id, type=Recipient.PERSONAL)
         recipient_user3 = Recipient.objects.get(type_id=user3.id, type=Recipient.PERSONAL)
@@ -1387,48 +1383,46 @@ class TestLoggingCountStats(AnalyticsTestCase):
         stream, _ = self.create_stream_with_recipient()
 
         invite_expires_in_minutes = 2 * 24 * 60
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_invite_users(
-                user,
-                ["user1@domain.tld", "user2@domain.tld"],
-                [stream],
-                invite_expires_in_minutes=invite_expires_in_minutes,
-            )
+        do_invite_users(
+            user,
+            ["user1@domain.tld", "user2@domain.tld"],
+            [stream],
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
         assertInviteCountEquals(2)
 
         # We currently send emails when re-inviting users that haven't
         # turned into accounts, so count them towards the total
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_invite_users(
-                user,
-                ["user1@domain.tld", "user2@domain.tld"],
-                [stream],
-                invite_expires_in_minutes=invite_expires_in_minutes,
-            )
+        do_invite_users(
+            user,
+            ["user1@domain.tld", "user2@domain.tld"],
+            [stream],
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
         assertInviteCountEquals(4)
 
         # Test mix of good and malformed invite emails
-        with self.assertRaises(InvitationError), mock.patch(
-            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
-        ):
+        try:
             do_invite_users(
                 user,
                 ["user3@domain.tld", "malformed"],
                 [stream],
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
+        except InvitationError:
+            pass
         assertInviteCountEquals(4)
 
         # Test inviting existing users
-        with self.assertRaises(InvitationError), mock.patch(
-            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
-        ):
+        try:
             do_invite_users(
                 user,
                 ["first@domain.tld", "user4@domain.tld"],
                 [stream],
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
+        except InvitationError:
+            pass
         assertInviteCountEquals(5)
 
         # Revoking invite should not give you credit
@@ -1438,8 +1432,7 @@ class TestLoggingCountStats(AnalyticsTestCase):
         assertInviteCountEquals(5)
 
         # Resending invite should cost you
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_resend_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
+        do_resend_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
         assertInviteCountEquals(6)
 
     def test_messages_read_hour(self) -> None:
@@ -1512,12 +1505,12 @@ class TestDeleteStats(AnalyticsTestCase):
         FillState.objects.create(property="test", end_time=self.TIME_ZERO, state=FillState.DONE)
 
         analytics = apps.get_app_config("analytics")
-        for table in analytics.models.values():
-            self.assertTrue(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertTrue(table.objects.exists())
 
         do_drop_all_analytics_tables()
-        for table in analytics.models.values():
-            self.assertFalse(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertFalse(table.objects.exists())
 
     def test_do_drop_single_stat(self) -> None:
         user = self.create_user()
@@ -1536,17 +1529,16 @@ class TestDeleteStats(AnalyticsTestCase):
         FillState.objects.create(property="to_save", end_time=self.TIME_ZERO, state=FillState.DONE)
 
         analytics = apps.get_app_config("analytics")
-        for table in analytics.models.values():
-            self.assertTrue(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertTrue(table.objects.exists())
 
         do_drop_single_stat("to_delete")
-        for table in analytics.models.values():
-            self.assertFalse(table._default_manager.filter(property="to_delete").exists())
-            self.assertTrue(table._default_manager.filter(property="to_save").exists())
+        for table in list(analytics.models.values()):
+            self.assertFalse(table.objects.filter(property="to_delete").exists())
+            self.assertTrue(table.objects.filter(property="to_save").exists())
 
 
 class TestActiveUsersAudit(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.user = self.create_user()
@@ -1729,7 +1721,6 @@ class TestActiveUsersAudit(AnalyticsTestCase):
 
 
 class TestRealmActiveHumans(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.stat = COUNT_STATS["realm_active_humans::day"]

analytics/tests/test_stats_views.py

@@ -2,11 +2,10 @@ from datetime import datetime, timedelta, timezone
 from typing import List, Optional
 
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, StreamCount, UserCount
+from analytics.models import FillState, RealmCount, UserCount
 from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
@@ -69,12 +68,10 @@ class TestStatsEndpoint(ZulipTestCase):
 
 
 class TestGetChartData(ZulipTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.realm = get_realm("zulip")
         self.user = self.example_user("hamlet")
-        self.stream_id = self.get_stream_id(self.get_streams(self.user)[0])
         self.login_user(self.user)
         self.end_times_hour = [
             ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
@@ -117,17 +114,6 @@ class TestGetChartData(ZulipTestCase):
             )
             for i, subgroup in enumerate(user_subgroups)
         )
-        StreamCount.objects.bulk_create(
-            StreamCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                stream_id=self.stream_id,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
         FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
 
     def test_number_of_humans(self) -> None:
@@ -193,20 +179,20 @@ class TestGetChartData(ZulipTestCase):
                 "everyone": {
                     "Public streams": self.data(100),
                     "Private streams": self.data(0),
-                    "Direct messages": self.data(101),
-                    "Group direct messages": self.data(0),
+                    "Private messages": self.data(101),
+                    "Group private messages": self.data(0),
                 },
                 "user": {
                     "Public streams": self.data(200),
                     "Private streams": self.data(201),
-                    "Direct messages": self.data(0),
-                    "Group direct messages": self.data(0),
+                    "Private messages": self.data(0),
+                    "Group private messages": self.data(0),
                 },
                 "display_order": [
-                    "Direct messages",
+                    "Private messages",
                     "Public streams",
                     "Private streams",
-                    "Group direct messages",
+                    "Group private messages",
                 ],
                 "result": "success",
             },
@@ -264,49 +250,6 @@ class TestGetChartData(ZulipTestCase):
             },
         )
 
-    def test_messages_sent_by_stream(self) -> None:
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
-        self.insert_data(stat, ["true", "false"], [])
-
-        result = self.client_get(
-            f"/json/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-        result = self.api_get(
-            self.example_user("polonius"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        self.assert_json_error(result, "Not allowed for guest users")
-
-        # Verify we correctly forbid access to stats of streams in other realms.
-        result = self.api_get(
-            self.mit_user("sipbtest"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-            subdomain="zephyr",
-        )
-        self.assert_json_error(result, "Invalid stream ID")
-
     def test_include_empty_subgroups(self) -> None:
         FillState.objects.create(
             property="realm_active_humans::day",
@@ -344,8 +287,8 @@ class TestGetChartData(ZulipTestCase):
             {
                 "Public streams": [0],
                 "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
+                "Private messages": [0],
+                "Group private messages": [0],
             },
         )
         self.assertEqual(
@@ -353,8 +296,8 @@ class TestGetChartData(ZulipTestCase):
             {
                 "Public streams": [0],
                 "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
+                "Private messages": [0],
+                "Group private messages": [0],
             },
         )
 

analytics/tests/test_support_views.py

@@ -4,15 +4,17 @@ from unittest import mock
 
 import orjson
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from corporate.lib.stripe import add_months, update_sponsorship_status
 from corporate.models import Customer, CustomerPlan, LicenseLedger, get_customer_by_realm
 from zerver.actions.invites import do_create_multiuse_invite_link
-from zerver.actions.realm_settings import do_change_realm_org_type, do_send_realm_reactivation_email
-from zerver.actions.user_settings import do_change_user_setting
+from zerver.actions.realm_settings import (
+    do_change_realm_org_type,
+    do_send_realm_reactivation_email,
+    do_set_realm_property,
+)
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm
+from zerver.lib.test_helpers import reset_emails_in_zulip_realm
 from zerver.models import (
     MultiuseInvite,
     PreregistrationUser,
@@ -26,59 +28,10 @@ from zerver.models import (
 if TYPE_CHECKING:
     from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
 
-import uuid
-
-from zilencer.models import RemoteZulipServer
-
-
-class TestRemoteServerSupportEndpoint(ZulipTestCase):
-    @override
-    def setUp(self) -> None:
-        super().setUp()
-
-        # Set up some initial example data.
-        for i in range(20):
-            hostname = f"zulip-{i}.example.com"
-            RemoteZulipServer.objects.create(
-                hostname=hostname, contact_email=f"admin@{hostname}", plan_type=1, uuid=uuid.uuid4()
-            )
-
-    def test_search(self) -> None:
-        self.login("cordelia")
-
-        result = self.client_get("/activity/remote/support")
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        # Iago is the user with the appropriate permissions to access this page.
-        self.login("iago")
-        assert self.example_user("iago").is_staff
-
-        result = self.client_get("/activity/remote/support")
-        self.assert_in_success_response(
-            [
-                'input type="text" name="q" class="input-xxlarge search-query" placeholder="hostname or contact email"'
-            ],
-            result,
-        )
-
-        result = self.client_get("/activity/remote/support", {"q": "zulip-1.example.com"})
-        self.assert_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-        self.assert_not_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
-
-        result = self.client_get("/activity/remote/support", {"q": "example.com"})
-        for i in range(20):
-            self.assert_in_success_response([f"<h3>zulip-{i}.example.com</h3>"], result)
-
-        result = self.client_get("/activity/remote/support", {"q": "admin@zulip-2.example.com"})
-        self.assert_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
-        self.assert_in_success_response(["<b>Contact email</b>: admin@zulip-2.example.com"], result)
-        self.assert_not_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-
 
 class TestSupportEndpoint(ZulipTestCase):
     def test_search(self) -> None:
-        reset_email_visibility_to_everyone_in_zulip_realm()
+        reset_emails_in_zulip_realm()
         lear_user = self.lear_user("king")
         lear_user.is_staff = True
         lear_user.save(update_fields=["is_staff"])
@@ -265,10 +218,10 @@ class TestSupportEndpoint(ZulipTestCase):
 
         self.login("iago")
 
-        do_change_user_setting(
-            self.example_user("hamlet"),
+        do_set_realm_property(
+            get_realm("zulip"),
             "email_address_visibility",
-            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+            Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY,
             acting_user=None,
         )
 
@@ -299,12 +252,6 @@ class TestSupportEndpoint(ZulipTestCase):
         check_hamlet_user_query_result(result)
         check_zulip_realm_query_result(result)
 
-        # Search should be case-insensitive:
-        assert self.example_email("hamlet") != self.example_email("hamlet").upper()
-        result = get_check_query_result(self.example_email("hamlet").upper(), 1)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
         result = get_check_query_result(lear_user.email, 1)
         check_lear_user_query_result(result)
         check_lear_realm_query_result(result)
@@ -358,9 +305,7 @@ class TestSupportEndpoint(ZulipTestCase):
             check_zulip_realm_query_result(result)
 
             email = self.nonreg_email("alice")
-            self.submit_realm_creation_form(
-                email, realm_subdomain="custom-test", realm_name="Zulip test"
-            )
+            self.client_post("/new/", {"email": email})
             result = get_check_query_result(email, 1)
             check_realm_creation_query_result(result, email)
 
@@ -400,7 +345,7 @@ class TestSupportEndpoint(ZulipTestCase):
         """
         Unspecified org type is special in that it is marked to not be shown
         on the registration page (because organitions are not meant to be able to choose it),
-        but should be correctly shown at the /support/ endpoint.
+        but should be correctly shown at the /support endpoint.
         """
         realm = get_realm("zulip")
 
@@ -649,29 +594,21 @@ class TestSupportEndpoint(ZulipTestCase):
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
         result = self.client_post(
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "zulip"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
         result = self.client_post(
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "lear"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
-        )
-
-        # Test renaming to a "reserved" subdomain
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "your-org"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain reserved. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
     def test_downgrade_realm(self) -> None:
@@ -691,7 +628,7 @@ class TestSupportEndpoint(ZulipTestCase):
                 "/activity/support",
                 {
                     "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "downgrade_at_billing_cycle_end",
+                    "downgrade_method": "downgrade_at_billing_cycle_end",
                 },
             )
             m.assert_called_once_with(get_realm("zulip"))
@@ -706,7 +643,7 @@ class TestSupportEndpoint(ZulipTestCase):
                 "/activity/support",
                 {
                     "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "downgrade_now_without_additional_licenses",
+                    "downgrade_method": "downgrade_now_without_additional_licenses",
                 },
             )
             m.assert_called_once_with(get_realm("zulip"))
@@ -722,7 +659,7 @@ class TestSupportEndpoint(ZulipTestCase):
                     "/activity/support",
                     {
                         "realm_id": f"{iago.realm_id}",
-                        "modify_plan": "downgrade_now_void_open_invoices",
+                        "downgrade_method": "downgrade_now_void_open_invoices",
                     },
                 )
                 m1.assert_called_once_with(get_realm("zulip"))
@@ -731,17 +668,6 @@ class TestSupportEndpoint(ZulipTestCase):
                     ["zulip downgraded and voided 1 open invoices"], result
                 )
 
-        with mock.patch("analytics.views.support.switch_realm_from_standard_to_plus_plan") as m:
-            result = self.client_post(
-                "/activity/support",
-                {
-                    "realm_id": f"{iago.realm_id}",
-                    "modify_plan": "upgrade_to_plus",
-                },
-            )
-            m.assert_called_once_with(get_realm("zulip"))
-            self.assert_in_success_response(["zulip upgraded to Plus"], result)
-
     def test_scrub_realm(self) -> None:
         cordelia = self.example_user("cordelia")
         lear_realm = get_realm("lear")
@@ -766,26 +692,3 @@ class TestSupportEndpoint(ZulipTestCase):
             result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
             self.assert_json_error(result, "Invalid parameters")
             m.assert_not_called()
-
-    def test_delete_user(self) -> None:
-        cordelia = self.example_user("cordelia")
-        hamlet = self.example_user("hamlet")
-        hamlet_email = hamlet.delivery_email
-        realm = get_realm("zulip")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.do_delete_user_preserving_messages") as m:
-            result = self.client_post(
-                "/activity/support",
-                {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id},
-            )
-            m.assert_called_once_with(hamlet)
-            self.assert_in_success_response([f"{hamlet_email} in zulip deleted"], result)

analytics/urls.py

@@ -4,36 +4,28 @@ from django.conf.urls import include
 from django.urls import path
 from django.urls.resolvers import URLPattern, URLResolver
 
-from analytics.views.installation_activity import (
-    get_installation_activity,
-    get_integrations_activity,
-)
+from analytics.views.installation_activity import get_installation_activity
 from analytics.views.realm_activity import get_realm_activity
-from analytics.views.remote_activity import get_remote_server_activity
 from analytics.views.stats import (
     get_chart_data,
     get_chart_data_for_installation,
     get_chart_data_for_realm,
     get_chart_data_for_remote_installation,
     get_chart_data_for_remote_realm,
-    get_chart_data_for_stream,
     stats,
     stats_for_installation,
     stats_for_realm,
     stats_for_remote_installation,
     stats_for_remote_realm,
 )
-from analytics.views.support import remote_servers_support, support
+from analytics.views.support import support
 from analytics.views.user_activity import get_user_activity
 from zerver.lib.rest import rest_path
 
 i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
     # Server admin (user_profile.is_staff) visible stats pages
     path("activity", get_installation_activity),
-    path("activity/remote", get_remote_server_activity),
-    path("activity/integrations", get_integrations_activity),
     path("activity/support", support, name="support"),
-    path("activity/remote/support", remote_servers_support, name="remote_servers_support"),
     path("realm_activity/<realm_str>/", get_realm_activity),
     path("user_activity/<user_profile_id>/", get_user_activity),
     path("stats/realm/<realm_str>/", stats_for_realm),
@@ -57,7 +49,6 @@ i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
 v1_api_and_json_patterns = [
     # get data for the graphs at /stats
     rest_path("analytics/chart_data", GET=get_chart_data),
-    rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
     rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
     rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
     rest_path(

analytics/views/activity_common.py

@@ -1,19 +1,16 @@
 import re
 import sys
 from datetime import datetime
-from typing import Any, Callable, Collection, Dict, List, Optional, Sequence, Union
-from urllib.parse import urlencode
+from html import escape
+from typing import Any, Collection, Dict, List, Optional, Sequence
 
 from django.conf import settings
-from django.db import connection
 from django.db.backends.utils import CursorWrapper
 from django.template import loader
 from django.urls import reverse
-from markupsafe import Markup
-from psycopg2.sql import Composable
+from markupsafe import Markup as mark_safe
 
-from zerver.lib.url_encoding import append_url_query_string
-from zerver.models import UserActivity, get_realm
+from zerver.models import UserActivity
 
 if sys.version_info < (3, 9):  # nocoverage
     from backports import zoneinfo
@@ -30,6 +27,7 @@ if settings.BILLING_ENABLED:
 def make_table(
     title: str, cols: Sequence[str], rows: Sequence[Any], has_row_class: bool = False
 ) -> str:
+
     if not has_row_class:
 
         def fix_row(row: Any) -> Dict[str, Any]:
@@ -47,50 +45,8 @@ def make_table(
     return content
 
 
-def get_page(
-    query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
-) -> Dict[str, str]:
-    cursor = connection.cursor()
-    cursor.execute(query)
-    rows = cursor.fetchall()
-    rows = list(map(list, rows))
-    cursor.close()
-
-    def fix_rows(
-        i: int, fixup_func: Union[Callable[[str], Markup], Callable[[datetime], str]]
-    ) -> None:
-        for row in rows:
-            row[i] = fixup_func(row[i])
-
-    total_row = []
-    for i, col in enumerate(cols):
-        if col == "Realm":
-            fix_rows(i, realm_activity_link)
-        elif col in ["Last time", "Last visit"]:
-            fix_rows(i, format_date_for_activity_reports)
-        elif col == "Hostname":
-            for row in rows:
-                row[i] = remote_installation_stats_link(row[0], row[i])
-        if len(totals_columns) > 0:
-            if i == 0:
-                total_row.append("Total")
-            elif i in totals_columns:
-                total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
-            else:
-                total_row.append("")
-    if len(totals_columns) > 0:
-        rows.insert(0, total_row)
-
-    content = make_table(title, cols, rows)
-
-    return dict(
-        content=content,
-        title=title,
-    )
-
-
 def dictfetchall(cursor: CursorWrapper) -> List[Dict[str, Any]]:
-    """Returns all rows from a cursor as a dict"""
+    "Returns all rows from a cursor as a dict"
     desc = cursor.description
     return [dict(zip((col[0] for col in desc), row)) for row in cursor.fetchall()]
 
@@ -102,46 +58,36 @@ def format_date_for_activity_reports(date: Optional[datetime]) -> str:
         return ""
 
 
-def user_activity_link(email: str, user_profile_id: int) -> Markup:
+def user_activity_link(email: str, user_profile_id: int) -> mark_safe:
     from analytics.views.user_activity import get_user_activity
 
     url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
-    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
+    email_link = f'<a href="{escape(url)}">{escape(email)}</a>'
+    return mark_safe(email_link)
 
 
-def realm_activity_link(realm_str: str) -> Markup:
+def realm_activity_link(realm_str: str) -> mark_safe:
     from analytics.views.realm_activity import get_realm_activity
 
     url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
+    realm_link = f'<a href="{escape(url)}">{escape(realm_str)}</a>'
+    return mark_safe(realm_link)
 
 
-def realm_stats_link(realm_str: str) -> Markup:
+def realm_stats_link(realm_str: str) -> mark_safe:
     from analytics.views.stats import stats_for_realm
 
     url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
+    stats_link = f'<a href="{escape(url)}"><i class="fa fa-pie-chart"></i>{escape(realm_str)}</a>'
+    return mark_safe(stats_link)
 
 
-def realm_support_link(realm_str: str) -> Markup:
-    support_url = reverse("support")
-    query = urlencode({"q": realm_str})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
-
-
-def realm_url_link(realm_str: str) -> Markup:
-    url = get_realm(realm_str).uri
-    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
-
-
-def remote_installation_stats_link(server_id: int, hostname: str) -> Markup:
+def remote_installation_stats_link(server_id: int, hostname: str) -> mark_safe:
     from analytics.views.stats import stats_for_remote_installation
 
     url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a> {hostname}').format(
-        url=url, hostname=hostname
-    )
+    stats_link = f'<a href="{escape(url)}"><i class="fa fa-pie-chart"></i>{escape(hostname)}</a>'
+    return mark_safe(stats_link)
 
 
 def get_user_activity_summary(records: Collection[UserActivity]) -> Dict[str, Any]:

analytics/views/installation_activity.py

@@ -1,9 +1,8 @@
 import itertools
 import time
 from collections import defaultdict
-from contextlib import suppress
-from datetime import timedelta
-from typing import Dict, Optional, Tuple
+from datetime import datetime, timedelta
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from django.conf import settings
 from django.db import connection
@@ -11,23 +10,23 @@ from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 from django.template import loader
 from django.utils.timezone import now as timezone_now
-from markupsafe import Markup
-from psycopg2.sql import SQL
+from markupsafe import Markup as mark_safe
+from psycopg2.sql import SQL, Composable, Literal
 
 from analytics.lib.counts import COUNT_STATS
 from analytics.views.activity_common import (
     dictfetchall,
-    get_page,
+    format_date_for_activity_reports,
+    make_table,
     realm_activity_link,
     realm_stats_link,
-    realm_support_link,
-    realm_url_link,
+    remote_installation_stats_link,
 )
 from analytics.views.support import get_plan_name
 from zerver.decorator import require_server_admin
 from zerver.lib.request import has_request_variables
 from zerver.lib.timestamp import timestamp_to_datetime
-from zerver.models import Realm, UserActivityInterval, get_org_type_display_name
+from zerver.models import Realm, UserActivityInterval, UserProfile, get_org_type_display_name
 
 if settings.BILLING_ENABLED:
     from corporate.lib.stripe import (
@@ -36,27 +35,29 @@ if settings.BILLING_ENABLED:
     )
 
 
-def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
-    # To align with UTC days, we subtract an hour from end_time to
-    # get the start_time, since the hour that starts at midnight was
-    # on the previous day.
+def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
     query = SQL(
         """
         select
             r.string_id,
-            (now()::date - (end_time - interval '1 hour')::date) age,
-            coalesce(sum(value), 0) cnt
-        from zerver_realm r
-        join analytics_realmcount rc on r.id = rc.realm_id
+            (now()::date - date_sent::date) age,
+            count(*) cnt
+        from zerver_message m
+        join zerver_userprofile up on up.id = m.sender_id
+        join zerver_realm r on r.id = up.realm_id
+        join zerver_client c on c.id = m.sending_client_id
         where
-            property = 'messages_sent:is_bot:hour'
+            (not up.is_bot)
         and
-            subgroup = 'false'
+            date_sent > now()::date - interval '8 day'
         and
-            end_time > now()::date - interval '8 day' - interval '1 hour'
+            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
         group by
             r.string_id,
             age
+        order by
+            r.string_id,
+            age
     """
     )
     cursor = connection.cursor()
@@ -68,25 +69,25 @@ def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
     for row in rows:
         counts[row["string_id"]][row["age"]] = row["cnt"]
 
-    def format_count(cnt: int, style: Optional[str] = None) -> Markup:
-        if style is not None:
-            good_bad = style
-        elif cnt == min_cnt:
-            good_bad = "bad"
-        elif cnt == max_cnt:
-            good_bad = "good"
-        else:
-            good_bad = "neutral"
-
-        return Markup('<td class="number {good_bad}">{cnt}</td>').format(good_bad=good_bad, cnt=cnt)
-
     result = {}
     for string_id in counts:
         raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
         min_cnt = min(raw_cnts[1:])
         max_cnt = max(raw_cnts[1:])
 
-        cnts = format_count(raw_cnts[0], "neutral") + Markup().join(map(format_count, raw_cnts[1:]))
+        def format_count(cnt: int, style: Optional[str] = None) -> str:
+            if style is not None:
+                good_bad = style
+            elif cnt == min_cnt:
+                good_bad = "bad"
+            elif cnt == max_cnt:
+                good_bad = "good"
+            else:
+                good_bad = "neutral"
+
+            return f'<td class="number {good_bad}">{cnt}</td>'
+
+        cnts = format_count(raw_cnts[0], "neutral") + "".join(map(format_count, raw_cnts[1:]))
         result[string_id] = dict(cnts=cnts)
 
     return result
@@ -186,10 +187,19 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
     rows = dictfetchall(cursor)
     cursor.close()
 
+    # Fetch all the realm administrator users
+    realm_owners: Dict[str, List[str]] = defaultdict(list)
+    for up in UserProfile.objects.select_related("realm").filter(
+        role=UserProfile.ROLE_REALM_OWNER,
+        is_active=True,
+    ):
+        realm_owners[up.realm.string_id].append(up.delivery_email)
+
     for row in rows:
         row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
         row["age_days"] = int((now - row["date_created"]).total_seconds() / 86400)
         row["is_new"] = row["age_days"] < 12 * 7
+        row["realm_owner_emails"] = ", ".join(realm_owners[row["string_id"]])
 
     # get messages sent per day
     counts = get_realm_day_counts()
@@ -238,18 +248,21 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
         hours = minutes / 60.0
         total_hours += hours
         row["hours"] = str(int(hours))
-        with suppress(Exception):
+        try:
             row["hours_per_user"] = "{:.1f}".format(hours / row["dau_count"])
+        except Exception:
+            pass
 
     # formatting
     for row in rows:
-        row["realm_url"] = realm_url_link(row["string_id"])
         row["stats_link"] = realm_stats_link(row["string_id"])
-        row["support_link"] = realm_support_link(row["string_id"])
         row["string_id"] = realm_activity_link(row["string_id"])
 
     # Count active sites
-    num_active_sites = sum(row["dau_count"] >= 5 for row in rows)
+    def meets_goal(row: Dict[str, int]) -> bool:
+        return row["dau_count"] >= 5
+
+    num_active_sites = len(list(filter(meets_goal, rows)))
 
     # create totals
     total_dau_count = 0
@@ -268,10 +281,9 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
         org_type_string="",
         effective_rate="",
         arr=total_arr,
-        realm_url="",
         stats_link="",
-        support_link="",
         date_created_day="",
+        realm_owner_emails="",
         dau_count=total_dau_count,
         user_profile_count=total_user_profile_count,
         bot_count=total_bot_count,
@@ -286,19 +298,18 @@ def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
         dict(
             rows=rows,
             num_active_sites=num_active_sites,
-            utctime=now.strftime("%Y-%m-%d %H:%M %Z"),
+            utctime=now.strftime("%Y-%m-%d %H:%MZ"),
             billing_enabled=settings.BILLING_ENABLED,
         ),
     )
     return content
 
 
-def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
+def user_activity_intervals() -> Tuple[mark_safe, Dict[str, float]]:
     day_end = timestamp_to_datetime(time.time())
     day_start = day_end - timedelta(hours=24)
 
-    output = Markup()
-    output += "Per-user online duration for the last 24 hours:\n"
+    output = "Per-user online duration for the last 24 hours:\n"
     total_duration = timedelta(0)
 
     all_intervals = (
@@ -329,7 +340,7 @@ def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
 
     for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
         realm_duration = timedelta(0)
-        output += Markup("<hr>") + f"{string_id}\n"
+        output += f"<hr>{string_id}\n"
         for email, intervals in itertools.groupby(realm_intervals, by_email):
             duration = timedelta(0)
             for interval in intervals:
@@ -346,31 +357,165 @@ def user_activity_intervals() -> Tuple[Markup, Dict[str, float]]:
     output += f"\nTotal duration:                      {total_duration}\n"
     output += f"\nTotal duration in minutes:           {total_duration.total_seconds() / 60.}\n"
     output += f"Total duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}"
-    content = Markup("<pre>{}</pre>").format(output)
+    content = mark_safe("<pre>" + output + "</pre>")
     return content, realm_minutes
 
 
-@require_server_admin
-@has_request_variables
-def get_installation_activity(request: HttpRequest) -> HttpResponse:
-    duration_content, realm_minutes = user_activity_intervals()
-    counts_content: str = realm_summary_table(realm_minutes)
-    data = [
-        ("Counts", counts_content),
-        ("Durations", duration_content),
-    ]
+def ad_hoc_queries() -> List[Dict[str, str]]:
+    def get_page(
+        query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
+    ) -> Dict[str, str]:
+        cursor = connection.cursor()
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        rows = list(map(list, rows))
+        cursor.close()
 
-    title = "Activity"
+        def fix_rows(
+            i: int, fixup_func: Union[Callable[[str], mark_safe], Callable[[datetime], str]]
+        ) -> None:
+            for row in rows:
+                row[i] = fixup_func(row[i])
 
-    return render(
-        request,
-        "analytics/activity.html",
-        context=dict(data=data, title=title, is_home=True),
+        total_row = []
+        for i, col in enumerate(cols):
+            if col == "Realm":
+                fix_rows(i, realm_activity_link)
+            elif col in ["Last time", "Last visit"]:
+                fix_rows(i, format_date_for_activity_reports)
+            elif col == "Hostname":
+                for row in rows:
+                    row[i] = remote_installation_stats_link(row[0], row[i])
+            if len(totals_columns) > 0:
+                if i == 0:
+                    total_row.append("Total")
+                elif i in totals_columns:
+                    total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
+                else:
+                    total_row.append("")
+        if len(totals_columns) > 0:
+            rows.insert(0, total_row)
+
+        content = make_table(title, cols, rows)
+
+        return dict(
+            content=content,
+            title=title,
+        )
+
+    pages = []
+
+    ###
+
+    for mobile_type in ["Android", "ZulipiOS"]:
+        title = f"{mobile_type} usage"
+
+        query: Composable = SQL(
+            """
+            select
+                realm.string_id,
+                up.id user_id,
+                client.name,
+                sum(count) as hits,
+                max(last_visit) as last_time
+            from zerver_useractivity ua
+            join zerver_client client on client.id = ua.client_id
+            join zerver_userprofile up on up.id = ua.user_profile_id
+            join zerver_realm realm on realm.id = up.realm_id
+            where
+                client.name like {mobile_type}
+            group by string_id, up.id, client.name
+            having max(last_visit) > now() - interval '2 week'
+            order by string_id, up.id, client.name
+        """
+        ).format(
+            mobile_type=Literal(mobile_type),
+        )
+
+        cols = [
+            "Realm",
+            "User id",
+            "Name",
+            "Hits",
+            "Last time",
+        ]
+
+        pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = "Desktop users"
+
+    query = SQL(
+        """
+        select
+            realm.string_id,
+            client.name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            client.name like 'desktop%%'
+        group by string_id, client.name
+        having max(last_visit) > now() - interval '2 week'
+        order by string_id, client.name
+    """
     )
 
+    cols = [
+        "Realm",
+        "Client",
+        "Hits",
+        "Last time",
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = "Integrations by realm"
+
+    query = SQL(
+        """
+        select
+            realm.string_id,
+            case
+                when query like '%%external%%' then split_part(query, '/', 5)
+                else client.name
+            end client_name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            (query in ('send_message_backend', '/api/v1/send_message')
+            and client.name not in ('Android', 'ZulipiOS')
+            and client.name not like 'test: Zulip%%'
+            )
+        or
+            query like '%%external%%'
+        group by string_id, client_name
+        having max(last_visit) > now() - interval '2 week'
+        order by string_id, client_name
+    """
+    )
+
+    cols = [
+        "Realm",
+        "Client",
+        "Hits",
+        "Last time",
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
 
-@require_server_admin
-def get_integrations_activity(request: HttpRequest) -> HttpResponse:
     title = "Integrations by client"
 
     query = SQL(
@@ -407,14 +552,71 @@ def get_integrations_activity(request: HttpRequest) -> HttpResponse:
         "Last time",
     ]
 
-    integrations_activity = get_page(query, cols, title)
+    pages.append(get_page(query, cols, title))
+
+    title = "Remote Zulip servers"
+
+    query = SQL(
+        """
+        with icount as (
+            select
+                server_id,
+                max(value) as max_value,
+                max(end_time) as max_end_time
+            from zilencer_remoteinstallationcount
+            where
+                property='active_users:is_bot:day'
+                and subgroup='false'
+            group by server_id
+            ),
+        remote_push_devices as (
+            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
+            group by server_id
+        )
+        select
+            rserver.id,
+            rserver.hostname,
+            rserver.contact_email,
+            max_value,
+            push_user_count,
+            max_end_time
+        from zilencer_remotezulipserver rserver
+        left join icount on icount.server_id = rserver.id
+        left join remote_push_devices on remote_push_devices.server_id = rserver.id
+        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
+    """
+    )
+
+    cols = [
+        "ID",
+        "Hostname",
+        "Contact email",
+        "Analytics users",
+        "Mobile users",
+        "Last update time",
+    ]
+
+    pages.append(get_page(query, cols, title, totals_columns=[3, 4]))
+
+    return pages
+
+
+@require_server_admin
+@has_request_variables
+def get_installation_activity(request: HttpRequest) -> HttpResponse:
+    duration_content, realm_minutes = user_activity_intervals()
+    counts_content: str = realm_summary_table(realm_minutes)
+    data = [
+        ("Counts", counts_content),
+        ("Durations", duration_content),
+    ]
+    for page in ad_hoc_queries():
+        data.append((page["title"], page["content"]))
+
+    title = "Activity"
 
     return render(
         request,
-        "analytics/activity_details_template.html",
-        context=dict(
-            data=integrations_activity["content"],
-            title=integrations_activity["title"],
-            is_home=False,
-        ),
+        "analytics/activity.html",
+        context=dict(data=data, title=title, is_home=True),
     )

analytics/views/realm_activity.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from typing import Any, Dict, List, Optional, Set, Tuple
 
 from django.db import connection
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils.timezone import now as timezone_now
@@ -13,7 +13,6 @@ from analytics.views.activity_common import (
     format_date_for_activity_reports,
     get_user_activity_summary,
     make_table,
-    realm_stats_link,
     user_activity_link,
 )
 from zerver.decorator import require_server_admin
@@ -163,13 +162,12 @@ def sent_messages_report(realm: str) -> str:
         "Bots",
     ]
 
-    # Uses index: zerver_message_realm_date_sent
     query = SQL(
         """
         select
             series.day::date,
-            user_messages.humans,
-            user_messages.bots
+            humans.cnt,
+            bots.cnt
         from (
             select generate_series(
                 (now()::date - interval '2 week'),
@@ -180,27 +178,45 @@ def sent_messages_report(realm: str) -> str:
         left join (
             select
                 date_sent::date date_sent,
-                count(*) filter (where not up.is_bot) as humans,
-                count(*) filter (where up.is_bot) as bots
+                count(*) cnt
             from zerver_message m
             join zerver_userprofile up on up.id = m.sender_id
             join zerver_realm r on r.id = up.realm_id
             where
                 r.string_id = %s
             and
-                date_sent > now() - interval '2 week'
+                (not up.is_bot)
             and
-                m.realm_id = r.id
+                date_sent > now() - interval '2 week'
             group by
                 date_sent::date
             order by
                 date_sent::date
-        ) user_messages on
-            series.day = user_messages.date_sent
+        ) humans on
+            series.day = humans.date_sent
+        left join (
+            select
+                date_sent::date date_sent,
+                count(*) cnt
+            from zerver_message m
+            join zerver_userprofile up on up.id = m.sender_id
+            join zerver_realm r on r.id = up.realm_id
+            where
+                r.string_id = %s
+            and
+                up.is_bot
+            and
+                date_sent > now() - interval '2 week'
+            group by
+                date_sent::date
+            order by
+                date_sent::date
+        ) bots on
+            series.day = bots.date_sent
     """
     )
     cursor = connection.cursor()
-    cursor.execute(query, [realm])
+    cursor.execute(query, [realm, realm])
     rows = cursor.fetchall()
     cursor.close()
 
@@ -236,10 +252,8 @@ def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
     data += [(page_title, content)]
 
     title = realm_str
-    realm_stats = realm_stats_link(realm_str)
-
     return render(
         request,
         "analytics/activity.html",
-        context=dict(data=data, realm_stats_link=realm_stats, title=title),
+        context=dict(data=data, realm_link=None, title=title),
     )

analytics/views/remote_activity.py

@@ -1,59 +0,0 @@
-from django.http import HttpRequest, HttpResponse
-from django.shortcuts import render
-from psycopg2.sql import SQL
-
-from analytics.views.activity_common import get_page
-from zerver.decorator import require_server_admin
-
-
-@require_server_admin
-def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
-    title = "Remote servers"
-
-    query = SQL(
-        """
-        with icount as (
-            select
-                server_id,
-                max(value) as max_value,
-                max(end_time) as max_end_time
-            from zilencer_remoteinstallationcount
-            where
-                property='active_users:is_bot:day'
-                and subgroup='false'
-            group by server_id
-            ),
-        remote_push_devices as (
-            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
-            group by server_id
-        )
-        select
-            rserver.id,
-            rserver.hostname,
-            rserver.contact_email,
-            max_value,
-            push_user_count,
-            max_end_time
-        from zilencer_remotezulipserver rserver
-        left join icount on icount.server_id = rserver.id
-        left join remote_push_devices on remote_push_devices.server_id = rserver.id
-        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
-    """
-    )
-
-    cols = [
-        "ID",
-        "Hostname",
-        "Contact email",
-        "Analytics users",
-        "Mobile users",
-        "Last update time",
-    ]
-
-    remote_servers = get_page(query, cols, title, totals_columns=[3, 4])
-
-    return render(
-        request,
-        "analytics/activity_details_template.html",
-        context=dict(data=remote_servers["content"], title=remote_servers["title"], is_home=False),
-    )

analytics/views/stats.py

@@ -4,13 +4,12 @@ from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
 
 from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils import translation
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
-from typing_extensions import TypeAlias
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
@@ -33,10 +32,9 @@ from zerver.lib.exceptions import JsonableError
 from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
-from zerver.lib.streams import access_stream_by_id
 from zerver.lib.timestamp import convert_to_UTC
 from zerver.lib.validator import to_non_negative_int
-from zerver.models import Client, Realm, Stream, UserProfile, get_realm
+from zerver.models import Client, Realm, UserProfile, get_realm
 
 if settings.ZILENCER_ENABLED:
     from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
@@ -51,35 +49,23 @@ def is_analytics_ready(realm: Realm) -> bool:
 def render_stats(
     request: HttpRequest,
     data_url_suffix: str,
-    realm: Optional[Realm],
-    *,
-    title: Optional[str] = None,
+    target_name: str,
     for_installation: bool = False,
     remote: bool = False,
     analytics_ready: bool = True,
 ) -> HttpResponse:
     assert request.user.is_authenticated
 
-    if realm is not None:
-        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
-        guest_users = UserProfile.objects.filter(
-            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
-        ).count()
-        space_used = realm.currently_used_upload_space_bytes()
-        if title:
-            pass
-        else:
-            title = realm.name or realm.string_id
-    else:
-        assert title
-        guest_users = None
-        space_used = None
+    # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
+    guest_users = UserProfile.objects.filter(
+        realm=request.user.realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
+    ).count()
 
     page_params = dict(
         data_url_suffix=data_url_suffix,
         for_installation=for_installation,
         remote=remote,
-        upload_space_used=space_used,
+        upload_space_used=request.user.realm.currently_used_upload_space_bytes(),
         guest_users=guest_users,
     )
 
@@ -95,9 +81,7 @@ def render_stats(
         request,
         "analytics/stats.html",
         context=dict(
-            target_name=title,
-            page_params=page_params,
-            analytics_ready=analytics_ready,
+            target_name=target_name, page_params=page_params, analytics_ready=analytics_ready
         ),
     )
 
@@ -110,7 +94,9 @@ def stats(request: HttpRequest) -> HttpResponse:
         # TODO: Make @zulip_login_required pass the UserProfile so we
         # can use @require_member_or_admin
         raise JsonableError(_("Not allowed for guest users"))
-    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
+    return render_stats(
+        request, "", realm.name or realm.string_id, analytics_ready=is_analytics_ready(realm)
+    )
 
 
 @require_server_admin
@@ -124,7 +110,7 @@ def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
     return render_stats(
         request,
         f"/realm/{realm_str}",
-        realm,
+        realm.name or realm.string_id,
         analytics_ready=is_analytics_ready(realm),
     )
 
@@ -139,8 +125,7 @@ def stats_for_remote_realm(
     return render_stats(
         request,
         f"/remote/{server.id}/realm/{remote_realm_id}",
-        None,
-        title=f"Realm {remote_realm_id} on server {server.hostname}",
+        f"Realm {remote_realm_id} on server {server.hostname}",
     )
 
 
@@ -157,21 +142,6 @@ def get_chart_data_for_realm(
     return get_chart_data(request, user_profile, realm=realm, **kwargs)
 
 
-@require_non_guest_user
-@has_request_variables
-def get_chart_data_for_stream(
-    request: HttpRequest, /, user_profile: UserProfile, stream_id: int
-) -> HttpResponse:
-    stream, ignored_sub = access_stream_by_id(
-        user_profile,
-        stream_id,
-        require_active=True,
-        allow_realm_admin=True,
-    )
-
-    return get_chart_data(request, user_profile, stream=stream)
-
-
 @require_server_admin_api
 @has_request_variables
 def get_chart_data_for_remote_realm(
@@ -196,8 +166,7 @@ def get_chart_data_for_remote_realm(
 
 @require_server_admin
 def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation", for_installation=True)
+    return render_stats(request, "/installation", "installation", True)
 
 
 @require_server_admin
@@ -207,10 +176,9 @@ def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -
     return render_stats(
         request,
         f"/remote/{server.id}/installation",
-        None,
-        title=f"remote installation {server.hostname}",
-        for_installation=True,
-        remote=True,
+        f"remote installation {server.hostname}",
+        True,
+        True,
     )
 
 
@@ -253,17 +221,13 @@ def get_chart_data(
     min_length: Optional[int] = REQ(converter=to_non_negative_int, default=None),
     start: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
     end: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
-    # These last several parameters are only used by functions
-    # wrapping get_chart_data; the callers are responsible for
-    # parsing/validation/authorization for them.
     realm: Optional[Realm] = None,
     for_installation: bool = False,
     remote: bool = False,
     remote_realm_id: Optional[int] = None,
     server: Optional["RemoteZulipServer"] = None,
-    stream: Optional[Stream] = None,
 ) -> HttpResponse:
-    TableType: TypeAlias = Union[
+    TableType = Union[
         Type["RemoteInstallationCount"],
         Type[InstallationCount],
         Type["RemoteRealmCount"],
@@ -285,9 +249,7 @@ def get_chart_data(
         else:
             aggregate_table = RealmCount
 
-    tables: Union[
-        Tuple[TableType], Tuple[TableType, Type[UserCount]], Tuple[TableType, Type[StreamCount]]
-    ]
+    tables: Union[Tuple[TableType], Tuple[TableType, Type[UserCount]]]
 
     if chart_name == "number_of_humans":
         stats = [
@@ -316,8 +278,8 @@ def get_chart_data(
             stats[0]: {
                 "public_stream": _("Public streams"),
                 "private_stream": _("Private streams"),
-                "private_message": _("Direct messages"),
-                "huddle_message": _("Group direct messages"),
+                "private_message": _("Private messages"),
+                "huddle_message": _("Group private messages"),
             }
         }
         labels_sort_function = lambda data: sort_by_totals(data["everyone"])
@@ -337,18 +299,8 @@ def get_chart_data(
         subgroup_to_label = {stats[0]: {None: "read"}}
         labels_sort_function = None
         include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_stream":
-        if stream is None:
-            raise JsonableError(
-                _("Missing stream for chart: {chart_name}").format(chart_name=chart_name)
-            )
-        stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
-        tables = (aggregate_table, StreamCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
     else:
-        raise JsonableError(_("Unknown chart name: {chart_name}").format(chart_name=chart_name))
+        raise JsonableError(_("Unknown chart name: {}").format(chart_name))
 
     # Most likely someone using our API endpoint. The /stats page does not
     # pass a start or end in its requests.
@@ -429,7 +381,6 @@ def get_chart_data(
         InstallationCount: "everyone",
         RealmCount: "everyone",
         UserCount: "user",
-        StreamCount: "everyone",
     }
     if settings.ZILENCER_ENABLED:
         aggregation_level[RemoteInstallationCount] = "everyone"
@@ -441,9 +392,6 @@ def get_chart_data(
         RealmCount: realm.id,
         UserCount: user_profile.id,
     }
-    if stream is not None:
-        id_value[StreamCount] = stream.id
-
     if settings.ZILENCER_ENABLED:
         if server is not None:
             id_value[RemoteInstallationCount] = server.id
@@ -474,7 +422,8 @@ def get_chart_data(
 
 
 def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
-    totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
+    totals = [(sum(values), label) for label, values in value_arrays.items()]
+    totals.sort(reverse=True)
     return [label for total, label in totals]
 
 
@@ -500,17 +449,17 @@ CountT = TypeVar("CountT", bound=BaseCount)
 
 def table_filtered_to_id(table: Type[CountT], key_id: int) -> QuerySet[CountT]:
     if table == RealmCount:
-        return table._default_manager.filter(realm_id=key_id)
+        return table.objects.filter(realm_id=key_id)
     elif table == UserCount:
-        return table._default_manager.filter(user_id=key_id)
+        return table.objects.filter(user_id=key_id)
     elif table == StreamCount:
-        return table._default_manager.filter(stream_id=key_id)
+        return table.objects.filter(stream_id=key_id)
     elif table == InstallationCount:
-        return table._default_manager.all()
+        return table.objects.all()
     elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
-        return table._default_manager.filter(server_id=key_id)
+        return table.objects.filter(server_id=key_id)
     elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
-        return table._default_manager.filter(realm_id=key_id)
+        return table.objects.filter(realm_id=key_id)
     else:
         raise AssertionError(f"Unknown table: {table}")
 
@@ -542,10 +491,10 @@ def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[
     for label, array in value_arrays.items():
         mapped_label = client_label_map(label)
         if mapped_label in mapped_arrays:
-            for i in range(len(array)):
+            for i in range(0, len(array)):
                 mapped_arrays[mapped_label][i] += value_arrays[label][i]
         else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
+            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
     return mapped_arrays
 
 

analytics/views/support.py

@@ -1,5 +1,4 @@
 import urllib
-from contextlib import suppress
 from dataclasses import dataclass
 from datetime import timedelta
 from decimal import Decimal
@@ -9,7 +8,6 @@ from urllib.parse import urlencode
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import URLValidator
-from django.db.models import Q
 from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import reverse
@@ -27,7 +25,6 @@ from zerver.actions.realm_settings import (
     do_scrub_realm,
     do_send_realm_reactivation_email,
 )
-from zerver.actions.users import do_delete_user_preserving_messages
 from zerver.decorator import require_server_admin
 from zerver.forms import check_subdomain_available
 from zerver.lib.exceptions import JsonableError
@@ -37,20 +34,15 @@ from zerver.lib.subdomains import get_subdomain_from_hostname
 from zerver.lib.validator import check_bool, check_string_in, to_decimal, to_non_negative_int
 from zerver.models import (
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     Realm,
     RealmReactivationStatus,
     UserProfile,
     get_org_type_display_name,
     get_realm,
-    get_user_profile_by_id,
 )
 from zerver.views.invite import get_invitee_emails_set
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import RemoteZulipServer
-
 if settings.BILLING_ENABLED:
     from corporate.lib.stripe import approve_sponsorship as do_approve_sponsorship
     from corporate.lib.stripe import (
@@ -60,7 +52,6 @@ if settings.BILLING_ENABLED:
         get_discount_for_realm,
         get_latest_seat_count,
         make_end_of_cycle_updates_if_needed,
-        switch_realm_from_standard_to_plus_plan,
         update_billing_method_of_current_plan,
         update_sponsorship_status,
         void_all_open_invoices,
@@ -128,11 +119,10 @@ def get_confirmations(
     return confirmation_dicts
 
 
-VALID_MODIFY_PLAN_METHODS = [
+VALID_DOWNGRADE_METHODS = [
     "downgrade_at_billing_cycle_end",
     "downgrade_now_without_additional_licenses",
     "downgrade_now_void_open_invoices",
-    "upgrade_to_plus",
 ]
 
 VALID_STATUS_VALUES = [
@@ -168,11 +158,10 @@ def support(
     ),
     sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
     approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
-    modify_plan: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
+    downgrade_method: Optional[str] = REQ(
+        default=None, str_validator=check_string_in(VALID_DOWNGRADE_METHODS)
     ),
     scrub_realm: bool = REQ(default=False, json_validator=check_bool),
-    delete_user_by_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
     query: Optional[str] = REQ("q", default=None),
     org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
 ) -> HttpResponse:
@@ -260,43 +249,31 @@ def support(
         elif approve_sponsorship:
             do_approve_sponsorship(realm, acting_user=acting_user)
             context["success_message"] = f"Sponsorship approved for {realm.string_id}"
-        elif modify_plan is not None:
-            if modify_plan == "downgrade_at_billing_cycle_end":
+        elif downgrade_method is not None:
+            if downgrade_method == "downgrade_at_billing_cycle_end":
                 downgrade_at_the_end_of_billing_cycle(realm)
                 context[
                     "success_message"
                 ] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
-            elif modify_plan == "downgrade_now_without_additional_licenses":
+            elif downgrade_method == "downgrade_now_without_additional_licenses":
                 downgrade_now_without_creating_additional_invoices(realm)
                 context[
                     "success_message"
                 ] = f"{realm.string_id} downgraded without creating additional invoices"
-            elif modify_plan == "downgrade_now_void_open_invoices":
+            elif downgrade_method == "downgrade_now_void_open_invoices":
                 downgrade_now_without_creating_additional_invoices(realm)
                 voided_invoices_count = void_all_open_invoices(realm)
                 context[
                     "success_message"
                 ] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
-            elif modify_plan == "upgrade_to_plus":
-                switch_realm_from_standard_to_plus_plan(realm)
-                context["success_message"] = f"{realm.string_id} upgraded to Plus"
         elif scrub_realm:
             do_scrub_realm(realm, acting_user=acting_user)
             context["success_message"] = f"{realm.string_id} scrubbed."
-        elif delete_user_by_id:
-            user_profile_for_deletion = get_user_profile_by_id(delete_user_by_id)
-            user_email = user_profile_for_deletion.delivery_email
-            assert user_profile_for_deletion.realm == realm
-            do_delete_user_preserving_messages(user_profile_for_deletion)
-            context["success_message"] = f"{user_email} in {realm.subdomain} deleted."
 
     if query:
         key_words = get_invitee_emails_set(query)
 
-        case_insensitive_users_q = Q()
-        for key_word in key_words:
-            case_insensitive_users_q |= Q(delivery_email__iexact=key_word)
-        users = set(UserProfile.objects.filter(case_insensitive_users_q))
+        users = set(UserProfile.objects.filter(delivery_email__in=key_words))
         realms = set(Realm.objects.filter(string_id__in=key_words))
 
         for key_word in key_words:
@@ -308,8 +285,10 @@ def support(
                 if parse_result.port:
                     hostname = f"{hostname}:{parse_result.port}"
                 subdomain = get_subdomain_from_hostname(hostname)
-                with suppress(Realm.DoesNotExist):
+                try:
                     realms.add(get_realm(subdomain))
+                except Realm.DoesNotExist:
+                    pass
             except ValidationError:
                 users.update(UserProfile.objects.filter(full_name__iexact=key_word))
 
@@ -325,20 +304,11 @@ def support(
             user.id for user in PreregistrationUser.objects.filter(email__in=key_words)
         ]
         confirmations += get_confirmations(
-            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION],
+            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION, Confirmation.REALM_CREATION],
             preregistration_user_ids,
             hostname=request.get_host(),
         )
 
-        preregistration_realm_ids = [
-            user.id for user in PreregistrationRealm.objects.filter(email__in=key_words)
-        ]
-        confirmations += get_confirmations(
-            [Confirmation.REALM_CREATION],
-            preregistration_realm_ids,
-            hostname=request.get_host(),
-        )
-
         multiuse_invite_ids = [
             invite.id for invite in MultiuseInvite.objects.filter(realm__in=realms)
         ]
@@ -409,43 +379,3 @@ def support(
     )
 
     return render(request, "analytics/support.html", context=context)
-
-
-def get_remote_servers_for_support(
-    email_to_search: Optional[str], hostname_to_search: Optional[str]
-) -> List["RemoteZulipServer"]:
-    if not email_to_search and not hostname_to_search:
-        return []
-
-    remote_servers_query = RemoteZulipServer.objects.order_by("id")
-    if email_to_search:
-        remote_servers_query = remote_servers_query.filter(contact_email__iexact=email_to_search)
-    elif hostname_to_search:
-        remote_servers_query = remote_servers_query.filter(hostname__icontains=hostname_to_search)
-
-    return list(remote_servers_query)
-
-
-@require_server_admin
-@has_request_variables
-def remote_servers_support(
-    request: HttpRequest, query: Optional[str] = REQ("q", default=None)
-) -> HttpResponse:
-    email_to_search = None
-    hostname_to_search = None
-    if query:
-        if "@" in query:
-            email_to_search = query
-        else:
-            hostname_to_search = query
-
-    remote_servers = get_remote_servers_for_support(
-        email_to_search=email_to_search, hostname_to_search=hostname_to_search
-    )
-    return render(
-        request,
-        "analytics/remote_server_support.html",
-        context=dict(
-            remote_servers=remote_servers,
-        ),
-    )

analytics/views/user_activity.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Tuple
 
 from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 
@@ -60,7 +60,7 @@ def raw_user_activity_table(records: QuerySet[UserActivity]) -> str:
 def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
     rows = []
     for k, v in user_summary.items():
-        if k in ("name", "user_profile_id"):
+        if k == "name" or k == "user_profile_id":
             continue
         client = k
         count = v["count"]

api_docs/construct-narrow.md

@@ -1,124 +0,0 @@
-# Construct a narrow
-
-A **narrow** is a set of filters for Zulip messages, that can be based
-on many different factors (like sender, stream, topic, search
-keywords, etc.). Narrows are used in various places in the the Zulip
-API (most importantly, in the API for fetching messages).
-
-It is simplest to explain the algorithm for encoding a search as a
-narrow using a single example. Consider the following search query
-(written as it would be entered in the Zulip web app's search box).
-It filters for messages sent to stream `announce`, not sent by
-`iago@zulip.com`, and containing the words `cool` and `sunglasses`:
-
-```
-stream:announce -sender:iago@zulip.com cool sunglasses
-```
-
-This query would be JSON-encoded for use in the Zulip API using JSON
-as a list of simple objects, as follows:
-
-```json
-[
-    {
-        "operator": "stream",
-        "operand": "announce"
-    },
-    {
-        "operator": "sender",
-        "operand": "iago@zulip.com",
-        "negated": true
-    },
-    {
-        "operator": "search",
-        "operand": "cool sunglasses"
-    }
-]
-```
-
-The Zulip help center article on [searching for messages](/help/search-for-messages)
-documents the majority of the search/narrow options supported by the
-Zulip API.
-
-Note that many narrows, including all that lack a `stream` or `streams`
-operator, search the current user's personal message history. See
-[searching shared history](/help/search-for-messages#searching-shared-history)
-for details.
-
-**Changes**: In Zulip 7.0 (feature level 177), support was added
-for three filters related to direct messages: `is:dm`, `dm` and
-`dm-including`. The `dm` operator replaced and deprecated the
-`pm-with` operator. The `is:dm` filter replaced and deprecated
-the `is:private` filter. The `dm-including` operator replaced and
-deprecated the `group-pm-with` operator.
-
-The `dm-including` and `group-pm-with` operators return slightly
-different results. For example, `dm-including:1234` returns all
-direct messages (1-on-1 and group) that include the current user
-and the user with the unique user ID of `1234`. On the other hand,
-`group-pm-with:1234` returned only group direct messages that included
-the current user and the user with the unique user ID of `1234`.
-
-Both `dm` and `is:dm` are aliases of `pm-with` and `is:private`
-respectively, and return the same exact results that the deprecated
-filters did.
-
-## Narrows that use IDs
-
-### Message IDs
-
-The `near` and `id` operators, documented in the help center, use message
-IDs for their operands.
-
-* `near:12345`: Search messages around the message with ID `12345`.
-* `id:12345`: Search for only message with ID `12345`.
-
-The message ID operand for the `id` operator may be encoded as either a
-number or a string. The message ID operand for the `near` operator must
-be encoded as a string.
-
-**Changes**: Prior to Zulip 8.0 (feature level 194), the message ID
-operand for the `id` operator needed to be encoded as a string.
-
-
-```json
-[
-    {
-        "operator": "id",
-        "operand": 12345
-    }
-]
-```
-
-### Stream and user IDs
-
-There are a few additional narrow/search options (new in Zulip 2.1)
-that use either stream IDs or user IDs that are not documented in the
-help center because they are primarily useful to API clients:
-
-* `stream:1234`: Search messages sent to the stream with ID `1234`.
-* `sender:1234`: Search messages sent by user ID `1234`.
-* `dm:1234`: Search the direct message conversation between
-  you and user ID `1234`.
-* `dm:1234,5678`: Search the direct message conversation between
-  you, user ID `1234`, and user ID `5678`.
-* `dm-including:1234`: Search all direct messages (1-on-1 and group)
-  that include you and user ID `1234`.
-
-The operands for these search options must be encoded either as an
-integer ID or a JSON list of integer IDs. For example, to query
-messages sent by a user 1234 to a direct message thread with yourself,
-user 1234, and user 5678, the correct JSON-encoded query is:
-
-```json
-[
-    {
-        "operator": "dm",
-        "operand": [1234, 5678]
-    },
-    {
-        "operator": "sender",
-        "operand": 1234
-    }
-]
-```

api_docs/create-scheduled-message.md

@@ -1,49 +0,0 @@
-{generate_api_header(/scheduled_messages:post)}
-
-## Usage examples
-
-{start_tabs}
-
-{generate_code_example(python)|/scheduled_messages:post|example}
-
-{generate_code_example(javascript)|/scheduled_messages:post|example}
-
-{tab|curl}
-
-``` curl
-# Create a scheduled stream message
-curl -X POST {{ api_url }}/v1/scheduled_messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=stream \
-    --data-urlencode to=9 \
-    --data-urlencode topic=Hello \
-    --data-urlencode 'content=Nice to meet everyone!' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-# Create a scheduled direct message
-curl -X POST {{ api_url }}/v1/messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=direct \
-    --data-urlencode 'to=[9, 10]' \
-    --data-urlencode 'content=Can we meet on Monday?' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-```
-
-{end_tabs}
-
-## Parameters
-
-{generate_api_arguments_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_parameter_description(/scheduled_messages:post)}
-
-## Response
-
-{generate_return_values_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_response_description(/scheduled_messages:post)}
-
-#### Example response(s)
-
-{generate_code_example|/scheduled_messages:post|fixture}

api_docs/http-headers.md

@@ -1,80 +0,0 @@
-# HTTP headers
-
-This page documents the HTTP headers used by the Zulip API.
-
-Most important is that API clients authenticate to the server using
-HTTP Basic authentication. If you're using the official [Python or
-JavaScript bindings](/api/installation-instructions), this is taken
-care of when you configure said bindings.
-
-Otherwise, see the `curl` example on each endpoint's documentation
-page, which details the request format.
-
-Documented below are additional HTTP headers and header conventions
-generally used by Zulip:
-
-## The `User-Agent` header
-
-Clients are not required to pass a `User-Agent` HTTP header, but we
-highly recommend doing so when writing an integration. It's easy to do
-and it can help save time when debugging issues related to an API
-client.
-
-If provided, the Zulip server will parse the `User-Agent` HTTP header
-in order to identify specific clients and integrations. This
-information is used by the server for logging, [usage
-statistics](/help/analytics), and on rare occasions, for
-backwards-compatibility logic to preserve support for older versions
-of official clients.
-
-Official Zulip clients and integrations use a `User-Agent` that starts
-with something like `ZulipMobile/20.0.103 `, encoding the name of the
-application and it's version.
-
-Zulip's official API bindings have reasonable defaults for
-`User-Agent`. For example, the official Zulip Python bindings have a
-default `User-Agent` starting with `ZulipPython/{version}`, where
-`version` is the version of the library.
-
-You can give your bot/integration its own name by passing the `client`
-parameter when initializing the Python bindings. For example, the
-official Zulip Nagios integration is initialized like this:
-
-``` python
-client = zulip.Client(
-    config_file=opts.config, client=f"ZulipNagios/{VERSION}"
-)
-```
-
-If you are working on an integration that you plan to share outside
-your organization, you can get help picking a good name in
-`#integrations` in the [Zulip development
-community](https://zulip.com/development-community).
-
-## Rate-limiting response headers
-
-To help clients avoid exceeding rate limits, Zulip sets the following
-HTTP headers in all API responses:
-
-* `X-RateLimit-Remaining`: The number of additional requests of this
-  type that the client can send before exceeding its limit.
-* `X-RateLimit-Limit`: The limit that would be applicable to a client
-  that had not made any recent requests of this type. This is useful
-  for designing a client's burst behavior so as to avoid ever reaching
-  a rate limit.
-* `X-RateLimit-Reset`: The time at which the client will no longer
-  have any rate limits applied to it (and thus could do a burst of
-  `X-RateLimit-Limit` requests).
-
-[Zulip's rate limiting rules are configurable][rate-limiting-rules],
-and can vary by server and over time. The default configuration
-currently limits:
-
-* Every user is limited to 200 total API requests per minute.
-* Separate, much lower limits for authentication/login attempts.
-
-When the Zulip server has configured multiple rate limits that apply
-to a given request, the values returned will be for the strictest
-limit.
-
-[rate-limiting-rules]: https://zulip.readthedocs.io/en/latest/production/security-model.html#rate-limiting

api_docs/rest-error-handling.md

@@ -1,34 +0,0 @@
-# Error handling
-
-Zulip's API will always return a JSON format response.
-The HTTP status code indicates whether the request was successful
-(200 = success, 40x = user error, 50x = server error).  Every response
-will contain at least two keys: `msg` (a human-readable error message)
-and `result`, which will be either `error` or `success` (this is
-redundant with the HTTP status code, but is convenient when printing
-responses while debugging).
-
-For some common errors, Zulip provides a `code` attribute.  Where
-present, clients should check `code`, rather than `msg`, when looking
-for specific error conditions, since the `msg` strings are
-internationalized (e.g. the server will send the error message
-translated into French if the user has a French locale).
-
-Each endpoint documents its own unique errors; documented below are
-errors common to many endpoints:
-
-{generate_code_example|/rest-error-handling:post|fixture}
-
-## Ignored Parameters
-
-In JSON success responses, all Zulip REST API endpoints may return
-an array of parameters sent in the request that are not supported
-by that specific endpoint.
-
-While this can be expected, e.g. when sending both current and legacy
-names for a parameter to a Zulip server of unknown version, this often
-indicates either a bug in the client implementation or an attempt to
-configure a new feature while connected to an older Zulip server that
-does not support said feature.
-
-{generate_code_example|/settings:patch|fixture}

api_docs/sidebar_index.md

@@ -1,27 +0,0 @@
-## Integrations
-
-* [Overview](/api/integrations-overview)
-* [Incoming webhook integrations](/api/incoming-webhooks-overview)
-* [Hello world walkthrough](/api/incoming-webhooks-walkthrough)
-* [Non-webhook integrations](/api/non-webhook-integrations)
-
-## Interactive bots (beta)
-
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)
-* [Outgoing webhooks](/api/outgoing-webhooks)
-
-## REST API
-
-* [Overview](/api/rest)
-* [Installation instructions](/api/installation-instructions)
-* [API keys](/api/api-keys)
-* [Configuring the Python bindings](/api/configuring-python-bindings)
-* [HTTP headers](/api/http-headers)
-* [Error handling](/api/rest-error-handling)
-* [Roles and permissions](/api/roles-and-permissions)
-* [Client libraries](/api/client-libraries)
-* [API changelog](/api/changelog)
-
-{!rest-endpoints.md!}

babel.config.js

@@ -0,0 +1,25 @@
+"use strict";
+
+module.exports = {
+    plugins: [
+        [
+            "formatjs",
+            {
+                additionalFunctionNames: ["$t", "$t_html"],
+                overrideIdFn: (id, defaultMessage) => defaultMessage,
+            },
+        ],
+    ],
+    presets: [
+        [
+            "@babel/preset-env",
+            {
+                corejs: "3.25",
+                shippedProposals: true,
+                useBuiltIns: "usage",
+            },
+        ],
+        "@babel/typescript",
+    ],
+    sourceType: "unambiguous",
+};

confirmation/migrations/0001_initial.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("contenttypes", "0001_initial"),
     ]

confirmation/migrations/0002_realmcreationkey.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0001_initial"),
     ]

confirmation/migrations/0003_emailchangeconfirmation.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0002_realmcreationkey"),
     ]

confirmation/migrations/0004_remove_confirmationmanager.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0003_emailchangeconfirmation"),
     ]

confirmation/migrations/0005_confirmation_realm.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0124_stream_enable_notifications"),
         ("confirmation", "0004_remove_confirmationmanager"),

confirmation/migrations/0006_realmcreationkey_presume_email_valid.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0005_confirmation_realm"),
     ]

confirmation/migrations/0007_add_indexes.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0006_realmcreationkey_presume_email_valid"),
     ]

confirmation/migrations/0008_confirmation_expiry_date.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0007_add_indexes"),
     ]

confirmation/migrations/0009_confirmation_expiry_date_backfill.py

@@ -5,7 +5,7 @@ from datetime import timedelta
 
 from django.conf import settings
 from django.db import migrations, transaction
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 

confirmation/migrations/0010_alter_confirmation_expiry_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0009_confirmation_expiry_date_backfill"),
     ]

confirmation/migrations/0011_alter_confirmation_expiry_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0010_alter_confirmation_expiry_date"),
     ]

confirmation/models.py

@@ -13,17 +13,15 @@ from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models import CASCADE
 from django.http import HttpRequest, HttpResponse
-from django.template.response import TemplateResponse
+from django.shortcuts import render
 from django.urls import reverse
 from django.utils.timezone import now as timezone_now
-from typing_extensions import TypeAlias, override
 
 from confirmation import settings as confirmation_settings
 from zerver.lib.types import UnspecifiedValue
 from zerver.models import (
     EmailChangeStatus,
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     Realm,
     RealmReactivationStatus,
@@ -31,7 +29,7 @@ from zerver.models import (
 )
 
 
-class ConfirmationKeyError(Exception):
+class ConfirmationKeyException(Exception):
     WRONG_LENGTH = 1
     EXPIRED = 2
     DOES_NOT_EXIST = 3
@@ -42,13 +40,13 @@ class ConfirmationKeyError(Exception):
 
 
 def render_confirmation_key_error(
-    request: HttpRequest, exception: ConfirmationKeyError
+    request: HttpRequest, exception: ConfirmationKeyException
 ) -> HttpResponse:
-    if exception.error_type == ConfirmationKeyError.WRONG_LENGTH:
-        return TemplateResponse(request, "confirmation/link_malformed.html", status=404)
-    if exception.error_type == ConfirmationKeyError.EXPIRED:
-        return TemplateResponse(request, "confirmation/link_expired.html", status=404)
-    return TemplateResponse(request, "confirmation/link_does_not_exist.html", status=404)
+    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
+        return render(request, "confirmation/link_malformed.html", status=404)
+    if exception.error_type == ConfirmationKeyException.EXPIRED:
+        return render(request, "confirmation/link_expired.html", status=404)
+    return render(request, "confirmation/link_does_not_exist.html", status=404)
 
 
 def generate_key() -> str:
@@ -56,9 +54,8 @@ def generate_key() -> str:
     return b32encode(secrets.token_bytes(15)).decode().lower()
 
 
-ConfirmationObjT: TypeAlias = Union[
+ConfirmationObjT = Union[
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     EmailChangeStatus,
     UserProfile,
@@ -80,16 +77,16 @@ def get_object_from_key(
 
     # Confirmation keys used to be 40 characters
     if len(confirmation_key) not in (24, 40):
-        raise ConfirmationKeyError(ConfirmationKeyError.WRONG_LENGTH)
+        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
     try:
         confirmation = Confirmation.objects.get(
             confirmation_key=confirmation_key, type__in=confirmation_types
         )
     except Confirmation.DoesNotExist:
-        raise ConfirmationKeyError(ConfirmationKeyError.DOES_NOT_EXIST)
+        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)
 
     if confirmation.expiry_date is not None and timezone_now() > confirmation.expiry_date:
-        raise ConfirmationKeyError(ConfirmationKeyError.EXPIRED)
+        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)
 
     obj = confirmation.content_object
     assert obj is not None
@@ -99,10 +96,10 @@ def get_object_from_key(
     if hasattr(obj, "status") and obj.status in [used_value, revoked_value]:
         # Confirmations where the object has the status attribute are one-time use
         # and are marked after being used (or revoked).
-        raise ConfirmationKeyError(ConfirmationKeyError.EXPIRED)
+        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)
 
     if mark_object_used:
-        # MultiuseInvite objects do not use the STATUS_USED status, since they are
+        # MultiuseInvite objects have no status field, since they are
         # intended to be used more than once.
         assert confirmation.type != Confirmation.MULTIUSE_INVITE
         assert hasattr(obj, "status")
@@ -117,17 +114,12 @@ def create_confirmation_link(
     *,
     validity_in_minutes: Union[Optional[int], UnspecifiedValue] = UnspecifiedValue(),
     url_args: Mapping[str, str] = {},
-    realm_creation: bool = False,
 ) -> str:
     # validity_in_minutes is an override for the default values which are
     # determined by the confirmation_type - its main purpose is for use
     # in tests which may want to have control over the exact expiration time.
     key = generate_key()
-    if realm_creation:
-        realm = None
-    else:
-        assert not isinstance(obj, PreregistrationRealm)
-        realm = obj.realm
+    realm = obj.realm
 
     current_time = timezone_now()
     expiry_date = None
@@ -187,13 +179,12 @@ class Confirmation(models.Model):
     REALM_REACTIVATION = 8
     type = models.PositiveSmallIntegerField()
 
+    def __str__(self) -> str:
+        return f"<Confirmation: {self.content_object}>"
+
     class Meta:
         unique_together = ("type", "confirmation_key")
 
-    @override
-    def __str__(self) -> str:
-        return f"{self.content_object!r}"
-
 
 class ConfirmationType:
     def __init__(
@@ -249,10 +240,10 @@ def validate_key(creation_key: Optional[str]) -> Optional["RealmCreationKey"]:
     try:
         key_record = RealmCreationKey.objects.get(creation_key=creation_key)
     except RealmCreationKey.DoesNotExist:
-        raise RealmCreationKey.InvalidError
+        raise RealmCreationKey.Invalid()
     time_elapsed = timezone_now() - key_record.date_created
     if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600:
-        raise RealmCreationKey.InvalidError
+        raise RealmCreationKey.Invalid()
     return key_record
 
 
@@ -275,5 +266,5 @@ class RealmCreationKey(models.Model):
     # is theirs, and skip sending mail to it to confirm that.
     presume_email_valid = models.BooleanField(default=False)
 
-    class InvalidError(Exception):
+    class Invalid(Exception):
         pass

corporate/lib/registration.py

@@ -72,13 +72,17 @@ def check_spare_licenses_available_for_adding_new_users(
     realm: Realm, extra_non_guests_count: int = 0, extra_guests_count: int = 0
 ) -> None:
     plan = get_current_plan_by_realm(realm)
-    if plan is None or plan.automanage_licenses or plan.customer.exempt_from_license_number_check:
+    if (
+        plan is None
+        or plan.automanage_licenses
+        or plan.customer.exempt_from_from_license_number_check
+    ):
         return
 
     if plan.licenses() < get_seat_count(
         realm, extra_non_guests_count=extra_non_guests_count, extra_guests_count=extra_guests_count
     ):
-        raise LicenseLimitError
+        raise LicenseLimitError()
 
 
 def check_spare_licenses_available_for_registering_new_user(

corporate/lib/stripe.py

@@ -7,6 +7,7 @@ from decimal import Decimal
 from functools import wraps
 from typing import Any, Callable, Dict, Generator, Optional, Tuple, TypeVar, Union
 
+import orjson
 import stripe
 from django.conf import settings
 from django.core.signing import Signer
@@ -55,8 +56,6 @@ DEFAULT_INVOICE_DAYS_UNTIL_DUE = 30
 # The version of Stripe API the billing system supports.
 STRIPE_API_VERSION = "2020-08-27"
 
-stripe.api_version = STRIPE_API_VERSION
-
 
 def get_latest_seat_count(realm: Realm) -> int:
     return get_seat_count(realm, extra_non_guests_count=0, extra_guests_count=0)
@@ -96,31 +95,23 @@ def unsign_string(signed_string: str, salt: str) -> str:
     return signer.unsign(signed_string)
 
 
-def validate_licenses(
-    charge_automatically: bool,
-    licenses: Optional[int],
-    seat_count: int,
-    exempt_from_license_number_check: bool,
-) -> None:
+def validate_licenses(charge_automatically: bool, licenses: Optional[int], seat_count: int) -> None:
     min_licenses = seat_count
     max_licenses = None
     if not charge_automatically:
         min_licenses = max(seat_count, MIN_INVOICED_LICENSES)
         max_licenses = MAX_INVOICED_LICENSES
 
-    if licenses is None or (not exempt_from_license_number_check and licenses < min_licenses):
+    if licenses is None or licenses < min_licenses:
         raise BillingError(
-            "not enough licenses",
-            _("You must invoice for at least {min_licenses} users.").format(
-                min_licenses=min_licenses
-            ),
+            "not enough licenses", _("You must invoice for at least {} users.").format(min_licenses)
         )
 
     if max_licenses is not None and licenses > max_licenses:
         message = _(
-            "Invoices with more than {max_licenses} licenses can't be processed from this page. To"
-            " complete the upgrade, please contact {email}."
-        ).format(max_licenses=max_licenses, email=settings.ZULIP_ADMINISTRATOR)
+            "Invoices with more than {} licenses can't be processed from this page. To complete "
+            "the upgrade, please contact {}."
+        ).format(max_licenses, settings.ZULIP_ADMINISTRATOR)
         raise BillingError("too many licenses", message)
 
 
@@ -257,13 +248,13 @@ class UpgradeWithExistingPlanError(BillingError):
         )
 
 
-class InvalidBillingScheduleError(Exception):
+class InvalidBillingSchedule(Exception):
     def __init__(self, billing_schedule: int) -> None:
         self.message = f"Unknown billing_schedule: {billing_schedule}"
         super().__init__(self.message)
 
 
-class InvalidTierError(Exception):
+class InvalidTier(Exception):
     def __init__(self, tier: int) -> None:
         self.message = f"Unknown tier: {tier}"
         super().__init__(self.message)
@@ -488,10 +479,12 @@ def make_end_of_cycle_updates_if_needed(
                 realm=realm,
                 event_time=event_time,
                 event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN,
-                extra_data={
-                    "monthly_plan_id": plan.id,
-                    "annual_plan_id": new_plan.id,
-                },
+                extra_data=orjson.dumps(
+                    {
+                        "monthly_plan_id": plan.id,
+                        "annual_plan_id": new_plan.id,
+                    }
+                ).decode(),
             )
             return new_plan, new_plan_ledger_entry
 
@@ -577,16 +570,16 @@ def get_price_per_license(
         elif billing_schedule == CustomerPlan.MONTHLY:
             price_per_license = 800
         else:  # nocoverage
-            raise InvalidBillingScheduleError(billing_schedule)
+            raise InvalidBillingSchedule(billing_schedule)
     elif tier == CustomerPlan.PLUS:
         if billing_schedule == CustomerPlan.ANNUAL:
             price_per_license = 16000
         elif billing_schedule == CustomerPlan.MONTHLY:
             price_per_license = 1600
         else:  # nocoverage
-            raise InvalidBillingScheduleError(billing_schedule)
+            raise InvalidBillingSchedule(billing_schedule)
     else:
-        raise InvalidTierError(tier)
+        raise InvalidTier(tier)
 
     if discount is not None:
         price_per_license = calculate_discounted_price_per_license(price_per_license, discount)
@@ -609,7 +602,7 @@ def compute_plan_parameters(
     elif billing_schedule == CustomerPlan.MONTHLY:
         period_end = add_months(billing_cycle_anchor, 1)
     else:  # nocoverage
-        raise InvalidBillingScheduleError(billing_schedule)
+        raise InvalidBillingSchedule(billing_schedule)
 
     price_per_license = get_price_per_license(tier, billing_schedule, discount)
 
@@ -624,6 +617,12 @@ def compute_plan_parameters(
     return billing_cycle_anchor, next_invoice_date, period_end, price_per_license
 
 
+def decimal_to_float(obj: object) -> object:
+    if isinstance(obj, Decimal):
+        return float(obj)
+    raise TypeError  # nocoverage
+
+
 def is_free_trial_offer_enabled() -> bool:
     return settings.FREE_TRIAL_DAYS not in (None, 0)
 
@@ -637,7 +636,7 @@ def ensure_realm_does_not_have_active_plan(realm: Realm) -> None:
             "Upgrade of %s failed because of existing active plan.",
             realm.string_id,
         )
-        raise UpgradeWithExistingPlanError
+        raise UpgradeWithExistingPlanError()
 
 
 @transaction.atomic
@@ -649,7 +648,7 @@ def do_change_remote_server_plan_type(remote_server: RemoteZulipServer, plan_typ
         event_type=RealmAuditLog.REMOTE_SERVER_PLAN_TYPE_CHANGED,
         server=remote_server,
         event_time=timezone_now(),
-        extra_data={"old_value": old_value, "new_value": plan_type},
+        extra_data=str({"old_value": old_value, "new_value": plan_type}),
     )
 
 
@@ -657,8 +656,8 @@ def do_change_remote_server_plan_type(remote_server: RemoteZulipServer, plan_typ
 def do_deactivate_remote_server(remote_server: RemoteZulipServer) -> None:
     if remote_server.deactivated:
         billing_logger.warning(
-            "Cannot deactivate remote server with ID %d, server has already been deactivated.",
-            remote_server.id,
+            f"Cannot deactivate remote server with ID {remote_server.id}, "
+            "server has already been deactivated."
         )
         return
 
@@ -702,12 +701,9 @@ def process_initial_upgrade(
     # TODO: The correctness of this relies on user creation, deactivation, etc being
     # in a transaction.atomic() with the relevant RealmAuditLog entries
     with transaction.atomic():
-        if customer.exempt_from_license_number_check:
-            billed_licenses = licenses
-        else:
-            # billed_licenses can be greater than licenses if users are added between the start of
-            # this function (process_initial_upgrade) and now
-            billed_licenses = max(get_latest_seat_count(realm), licenses)
+        # billed_licenses can greater than licenses if users are added between the start of
+        # this function (process_initial_upgrade) and now
+        billed_licenses = max(get_latest_seat_count(realm), licenses)
         plan_params = {
             "automanage_licenses": automanage_licenses,
             "charge_automatically": charge_automatically,
@@ -736,7 +732,7 @@ def process_initial_upgrade(
             acting_user=user,
             event_time=billing_cycle_anchor,
             event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED,
-            extra_data=plan_params,
+            extra_data=orjson.dumps(plan_params, default=decimal_to_float).decode(),
         )
 
     if not free_trial:
@@ -782,16 +778,14 @@ def update_license_ledger_for_manual_plan(
 ) -> None:
     if licenses is not None:
         assert plan.customer.realm is not None
-        if not plan.customer.exempt_from_license_number_check:
-            assert get_latest_seat_count(plan.customer.realm) <= licenses
+        assert get_latest_seat_count(plan.customer.realm) <= licenses
         assert licenses > plan.licenses()
         LicenseLedger.objects.create(
             plan=plan, event_time=event_time, licenses=licenses, licenses_at_next_renewal=licenses
         )
     elif licenses_at_next_renewal is not None:
         assert plan.customer.realm is not None
-        if not plan.customer.exempt_from_license_number_check:
-            assert get_latest_seat_count(plan.customer.realm) <= licenses_at_next_renewal
+        assert get_latest_seat_count(plan.customer.realm) <= licenses_at_next_renewal
         LicenseLedger.objects.create(
             plan=plan,
             event_time=event_time,
@@ -941,9 +935,7 @@ def invoice_plan(plan: CustomerPlan, event_time: datetime) -> None:
     plan.save(update_fields=["next_invoice_date"])
 
 
-def invoice_plans_as_needed(event_time: Optional[datetime] = None) -> None:
-    if event_time is None:  # nocoverage
-        event_time = timezone_now()
+def invoice_plans_as_needed(event_time: datetime = timezone_now()) -> None:
     for plan in CustomerPlan.objects.filter(next_invoice_date__lte=event_time):
         invoice_plan(plan, event_time)
 
@@ -974,7 +966,7 @@ def attach_discount_to_realm(
         acting_user=acting_user,
         event_type=RealmAuditLog.REALM_DISCOUNT_CHANGED,
         event_time=timezone_now(),
-        extra_data={"old_discount": old_discount, "new_discount": discount},
+        extra_data=str({"old_discount": old_discount, "new_discount": discount}),
     )
 
 
@@ -989,7 +981,7 @@ def update_sponsorship_status(
         acting_user=acting_user,
         event_type=RealmAuditLog.REALM_SPONSORSHIP_PENDING_STATUS_CHANGED,
         event_time=timezone_now(),
-        extra_data={"sponsorship_pending": sponsorship_pending},
+        extra_data=str({"sponsorship_pending": sponsorship_pending}),
     )
 
 
@@ -1012,16 +1004,11 @@ def approve_sponsorship(realm: Realm, *, acting_user: Optional[UserProfile]) ->
     for user in realm.get_human_billing_admin_and_realm_owner_users():
         with override_language(user.default_language):
             # Using variable to make life easier for translators if these details change.
+            plan_name = "Zulip Cloud Standard"
+            emoji = ":tada:"
             message = _(
-                "Your organization's request for sponsored hosting has been approved! "
-                "You have been upgraded to {plan_name}, free of charge. {emoji}\n\n"
-                "If you could {begin_link}list Zulip as a sponsor on your website{end_link}, "
-                "we would really appreciate it!"
-            ).format(
-                plan_name="Zulip Cloud Standard",
-                emoji=":tada:",
-                begin_link="[",
-                end_link="](/help/linking-to-zulip-website)",
+                f"Your organization's request for sponsored hosting has been approved! {emoji}.\n"
+                f"You have been upgraded to {plan_name}, free of charge."
             )
             internal_send_private_message(notification_bot, user, message)
 
@@ -1198,10 +1185,6 @@ def switch_realm_from_standard_to_plus_plan(realm: Realm) -> None:
     standard_plan.next_invoice_date = plan_switch_time
     standard_plan.save(update_fields=["status", "next_invoice_date"])
 
-    from zerver.actions.realm_settings import do_change_realm_plan_type
-
-    do_change_realm_plan_type(realm, Realm.PLAN_TYPE_PLUS, acting_user=None)
-
     standard_plan_next_renewal_date = start_of_next_billing_cycle(standard_plan, plan_switch_time)
 
     standard_plan_last_renewal_ledger = (
@@ -1243,5 +1226,5 @@ def update_billing_method_of_current_plan(
             acting_user=acting_user,
             event_type=RealmAuditLog.REALM_BILLING_METHOD_CHANGED,
             event_time=timezone_now(),
-            extra_data={"charge_automatically": charge_automatically},
+            extra_data=str({"charge_automatically": charge_automatically}),
         )

corporate/lib/stripe_event_handler.py

@@ -1,5 +1,4 @@
 import logging
-from contextlib import suppress
 from typing import Any, Callable, Dict, Union
 
 import stripe
@@ -87,12 +86,14 @@ def handle_checkout_session_completed_event(
         session.payment_intent.status = PaymentIntent.PROCESSING
         session.payment_intent.last_payment_error = ()
         session.payment_intent.save(update_fields=["status", "last_payment_error"])
-        with suppress(stripe.error.CardError):
+        try:
             stripe.PaymentIntent.confirm(
                 session.payment_intent.stripe_payment_intent_id,
                 payment_method=payment_method,
                 off_session=True,
             )
+        except stripe.error.CardError:
+            pass
     elif session.type in [
         Session.FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE,
         Session.FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE,

corporate/migrations/0001_initial.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     initial = True
 
     dependencies = [

corporate/migrations/0002_customer_default_discount.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0001_initial"),
     ]

corporate/migrations/0003_customerplan.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0002_customer_default_discount"),
     ]

corporate/migrations/0004_licenseledger.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0003_customerplan"),
     ]

corporate/migrations/0005_customerplan_invoicing.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0004_licenseledger"),
     ]

corporate/migrations/0006_nullable_stripe_customer_id.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0005_customerplan_invoicing"),
     ]

corporate/migrations/0007_remove_deprecated_fields.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0006_nullable_stripe_customer_id"),
     ]

corporate/migrations/0008_nullable_next_invoice_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0007_remove_deprecated_fields"),
     ]

corporate/migrations/0009_customer_sponsorship_pending.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0008_nullable_next_invoice_date"),
     ]

corporate/migrations/0010_customerplan_exempt_from_from_license_number_check.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0009_customer_sponsorship_pending"),
     ]

corporate/migrations/0012_zulipsponsorshiprequest.py

@@ -6,6 +6,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ("zerver", "0333_alter_realm_org_type"),

corporate/migrations/0013_alter_zulipsponsorshiprequest_org_website.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0012_zulipsponsorshiprequest"),
     ]

corporate/migrations/0014_customerplan_end_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0013_alter_zulipsponsorshiprequest_org_website"),
     ]

corporate/migrations/0015_event_paymentintent_session.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("contenttypes", "0002_remove_content_type_name"),
         ("corporate", "0014_customerplan_end_date"),

corporate/migrations/0016_customer_add_remote_server_field.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zilencer", "0018_remoterealmauditlog"),
         ("zerver", "0370_realm_enable_spectator_access"),

corporate/migrations/0017_rename_exempt_from_from_license_number_check_customer_exempt_from_license_number_check.py

@@ -1,17 +0,0 @@
-# Generated by Django 4.2 on 2023-04-10 18:22
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("corporate", "0016_customer_add_remote_server_field"),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name="customer",
-            old_name="exempt_from_from_license_number_check",
-            new_name="exempt_from_license_number_check",
-        ),
-    ]