Release Zulip Server 6.0-beta1.

.codespellignore

@@ -21,8 +21,3 @@ vise
 falsy
 ro
 derails
-forin
-uper
-slac
-couldn
-ges

.eslintignore

@@ -4,12 +4,11 @@
 
 /docs/_build
 /static/generated
+/static/third
 /static/webpack-bundles
 /var/*
 !/var/puppeteer
 /var/puppeteer/*
 !/var/puppeteer/test_credentials.d.ts
-/web/generated
-/web/third
 /zulip-current-venv
 /zulip-py3-venv

.eslintrc.json

@@ -1,5 +1,4 @@
 {
-    "root": true,
     "env": {
         "es2020": true,
         "node": true
@@ -15,7 +14,6 @@
     ],
     "parser": "@babel/eslint-parser",
     "parserOptions": {
-        "requireConfigFile": false,
         "warnOnUnsupportedTypeScriptVersion": false,
         "sourceType": "unambiguous"
     },
@@ -50,12 +48,15 @@
         "import/extensions": "error",
         "import/first": "error",
         "import/newline-after-import": "error",
-        "import/no-cycle": ["error", {"ignoreExternal": true}],
-        "import/no-duplicates": "error",
         "import/no-self-import": "error",
-        "import/no-unresolved": "off",
         "import/no-useless-path-segments": "error",
-        "import/order": ["error", {"alphabetize": {"order": "asc"}, "newlines-between": "always"}],
+        "import/order": [
+            "error",
+            {
+                "alphabetize": {"order": "asc"},
+                "newlines-between": "always"
+            }
+        ],
         "import/unambiguous": "error",
         "lines-around-directive": "error",
         "new-cap": "error",
@@ -66,6 +67,7 @@
         "no-catch-shadow": "error",
         "no-constant-condition": ["error", {"checkLoops": false}],
         "no-div-regex": "error",
+        "no-duplicate-imports": "error",
         "no-else-return": "error",
         "no-eq-null": "error",
         "no-eval": "error",
@@ -94,18 +96,20 @@
         "no-undef-init": "error",
         "no-unneeded-ternary": ["error", {"defaultAssignment": false}],
         "no-unused-expressions": "error",
-        "no-unused-vars": [
-            "error",
-            {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
-        ],
+        "no-unused-vars": ["error", {"ignoreRestSiblings": true}],
         "no-use-before-define": ["error", {"functions": false}],
         "no-useless-concat": "error",
         "no-useless-constructor": "error",
         "no-var": "error",
-        "object-shorthand": ["error", "always", {"avoidExplicitReturnArrows": true}],
+        "object-shorthand": "error",
         "one-var": ["error", "never"],
         "prefer-arrow-callback": "error",
-        "prefer-const": ["error", {"ignoreReadBeforeAssign": true}],
+        "prefer-const": [
+            "error",
+            {
+                "ignoreReadBeforeAssign": true
+            }
+        ],
         "radix": "error",
         "sort-imports": ["error", {"ignoreDeclarationSort": true}],
         "spaced-comment": ["error", "always", {"markers": ["/"]}],
@@ -114,13 +118,15 @@
         "unicorn/explicit-length-check": "off",
         "unicorn/filename-case": "off",
         "unicorn/no-await-expression-member": "off",
-        "unicorn/no-negated-condition": "off",
+        "unicorn/no-nested-ternary": "off",
         "unicorn/no-null": "off",
         "unicorn/no-process-exit": "off",
         "unicorn/no-useless-undefined": "off",
+        "unicorn/number-literal-case": "off",
         "unicorn/numeric-separators-style": "off",
         "unicorn/prefer-module": "off",
         "unicorn/prefer-node-protocol": "off",
+        "unicorn/prefer-spread": "off",
         "unicorn/prefer-ternary": "off",
         "unicorn/prefer-top-level-await": "off",
         "unicorn/prevent-abbreviations": "off",
@@ -130,19 +136,19 @@
     },
     "overrides": [
         {
-            "files": ["web/tests/**"],
+            "files": ["frontend_tests/node_tests/**", "frontend_tests/zjsunit/**"],
             "rules": {
                 "no-jquery/no-selector-prop": "off"
             }
         },
         {
-            "files": ["web/e2e-tests/**"],
+            "files": ["frontend_tests/puppeteer_lib/**", "frontend_tests/puppeteer_tests/**"],
             "globals": {
                 "zulip_test": false
             }
         },
         {
-            "files": ["web/src/**"],
+            "files": ["static/js/**"],
             "globals": {
                 "StripeCheckout": false
             }
@@ -150,9 +156,7 @@
         {
             "files": ["**/*.ts"],
             "extends": [
-                "plugin:@typescript-eslint/recommended",
                 "plugin:@typescript-eslint/recommended-requiring-type-checking",
-                "plugin:@typescript-eslint/strict",
                 "plugin:import/typescript"
             ],
             "parserOptions": {
@@ -170,34 +174,37 @@
             },
             "rules": {
                 // Disable base rule to avoid conflict
+                "no-duplicate-imports": "off",
+                "no-unused-vars": "off",
+                "no-useless-constructor": "off",
                 "no-use-before-define": "off",
 
-                "@typescript-eslint/consistent-type-assertions": [
-                    "error",
-                    {"assertionStyle": "never"}
-                ],
-                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
+                "@typescript-eslint/array-type": "error",
+                "@typescript-eslint/consistent-type-assertions": "error",
                 "@typescript-eslint/consistent-type-imports": "error",
                 "@typescript-eslint/explicit-function-return-type": [
                     "error",
                     {"allowExpressions": true}
                 ],
                 "@typescript-eslint/member-ordering": "error",
+                "@typescript-eslint/no-duplicate-imports": "off",
+                "@typescript-eslint/no-explicit-any": "off",
+                "@typescript-eslint/no-extraneous-class": "error",
                 "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-unnecessary-condition": "off",
+                "@typescript-eslint/no-parameter-properties": "error",
                 "@typescript-eslint/no-unnecessary-qualifier": "error",
+                "@typescript-eslint/no-unused-vars": ["error", {"ignoreRestSiblings": true}],
                 "@typescript-eslint/no-unsafe-argument": "off",
                 "@typescript-eslint/no-unsafe-assignment": "off",
                 "@typescript-eslint/no-unsafe-call": "off",
                 "@typescript-eslint/no-unsafe-member-access": "off",
                 "@typescript-eslint/no-unsafe-return": "off",
-                "@typescript-eslint/no-unused-vars": [
-                    "error",
-                    {"args": "all", "argsIgnorePattern": "^_", "ignoreRestSiblings": true}
-                ],
-                "@typescript-eslint/no-use-before-define": ["error", {"functions": false}],
-                "@typescript-eslint/parameter-properties": "error",
+                "@typescript-eslint/no-use-before-define": "error",
+                "@typescript-eslint/no-useless-constructor": "error",
+                "@typescript-eslint/prefer-includes": "error",
+                "@typescript-eslint/prefer-string-starts-ends-with": "error",
                 "@typescript-eslint/promise-function-async": "error",
+                "@typescript-eslint/unified-signatures": "error",
                 "no-undef": "error"
             }
         },
@@ -208,7 +215,7 @@
             }
         },
         {
-            "files": ["web/e2e-tests/**", "web/tests/**"],
+            "files": ["frontend_tests/**"],
             "globals": {
                 "CSS": false,
                 "document": false,
@@ -223,7 +230,7 @@
             }
         },
         {
-            "files": ["web/debug-require.js"],
+            "files": ["tools/debug-require.js"],
             "env": {
                 "browser": true,
                 "es2020": false
@@ -237,27 +244,20 @@
             }
         },
         {
-            "files": ["web/shared/**", "web/src/**", "web/third/**"],
+            "files": ["static/**"],
             "env": {
                 "browser": true,
                 "node": false
             },
-            "globals": {
-                "ZULIP_VERSION": false
-            },
             "rules": {
                 "no-console": "error"
             },
             "settings": {
-                "import/resolver": {
-                    "webpack": {
-                        "config": "./web/webpack.config.ts"
-                    }
-                }
+                "import/resolver": "webpack"
             }
         },
         {
-            "files": ["web/shared/**"],
+            "files": ["static/shared/**"],
             "env": {
                 "browser": false,
                 "shared-node-browser": true
@@ -268,14 +268,13 @@
                     {
                         "zones": [
                             {
-                                "target": "./web/shared",
+                                "target": "./static/shared",
                                 "from": ".",
-                                "except": ["./node_modules", "./web/shared"]
+                                "except": ["./node_modules", "./static/shared"]
                             }
                         ]
                     }
-                ],
-                "unicorn/prefer-string-replace-all": "off"
+                ]
             }
         }
     ]

.github/ISSUE_TEMPLATE/1_discussed_on_czo.md

@@ -1,10 +0,0 @@
----
-name: Issue discussed in the Zulip development community
-about: Bug report, feature or improvement already discussed on chat.zulip.org.
----
-
-<!-- Issue description -->
-
-<!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
-
-CZO thread

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -1,17 +0,0 @@
----
-name: Bug report
-about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
-labels: ["bug"]
----
-
-<!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
-
-<!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
-
-**Zulip Server and web app version:**
-
-- [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 7.0+
-- [ ] Zulip Server 6.0+
-- [ ] Zulip Server 5.0 or older
-- [ ] Other or not sure

.github/ISSUE_TEMPLATE/3_feature_request.md

@@ -1,6 +0,0 @@
----
-name: Feature or improvement request
-about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
----
-
-<!-- Describe the proposal, including how it would help you or your organization. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Possible bug
-    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
-    about: Report unexpected behavior that may be a bug.
-  - name: Feature suggestion or feedback
-    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
-    about: Start a discussion about your idea for improving Zulip.
-  - name: Issue with running or upgrading a Zulip server
-    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
-    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
-  - name: Other support requests and sales questions
-    url: https://zulip.com/help/contact-support
-    about: Contact us — we're happy to help!

.github/pull_request_template.md

@@ -9,8 +9,7 @@ Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gi
 
 **Screenshots and screen captures:**
 
-<details>
-<summary>Self-review checklist</summary>
+**Self-review checklist**
 
 <!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
 https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
@@ -28,7 +27,7 @@ Communicate decisions, questions, and potential concerns.
 - [ ] Calls out remaining decisions and concerns.
 - [ ] Automated tests verify logic where appropriate.
 
-Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
+Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html)).
 
 - [ ] Each commit is a coherent idea.
 - [ ] Commit message(s) explain reasoning and motivation for changes.
@@ -40,4 +39,3 @@ Completed manual review and testing of the following:
 - [ ] Strings and tooltips.
 - [ ] End-to-end functionality of buttons, interactions and flows.
 - [ ] Corner cases, error conditions, and easily imagined bugs.
-</details>

.github/workflows/production-suite.yml

@@ -8,16 +8,17 @@ on:
     paths:
       - .github/workflows/production-suite.yml
       - "**/migrations/**"
+      - babel.config.js
       - manage.py
-      - pnpm-lock.yaml
+      - postcss.config.js
       - puppet/**
       - requirements/**
       - scripts/**
+      - static/assets/**
+      - static/third/**
       - tools/**
-      - web/babel.config.js
-      - web/postcss.config.js
-      - web/third/**
-      - web/webpack.config.ts
+      - webpack.config.ts
+      - yarn.lock
       - zerver/worker/queue_processors.py
       - zerver/lib/push_notifications.py
       - zerver/decorator.py
@@ -46,7 +47,6 @@ jobs:
     # the top explain how to build and upload these images.
     # Ubuntu 20.04 ships with Python 3.8.10.
     container: zulip/ci:focal
-
     steps:
       - name: Add required permissions
         run: |
@@ -68,15 +68,16 @@ jobs:
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
+      - name: Restore node_modules cache
         uses: actions/cache@v3
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-focal-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-focal-${{ hashFiles('package.json') }}-${{ hashFiles('yarn.lock') }}
+          restore-keys: v1-yarn-deps-focal
 
       - name: Restore python cache
         uses: actions/cache@v3
@@ -100,30 +101,13 @@ jobs:
         with:
           name: production-tarball
           path: /tmp/production-build
-          retention-days: 1
+          retention-days: 14
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message
 
   production_install:
     # This job installs the server release tarball built above on a
@@ -132,29 +116,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        extra-args: [""]
         include:
           # Docker images are built from 'tools/ci/Dockerfile'; the comments at
           # the top explain how to build and upload these images.
           - docker_image: zulip/ci:focal
-            name: Ubuntu 20.04 production install and PostgreSQL upgrade with pgroonga
+            name: Ubuntu 20.04 production install
             os: focal
-            extra-args: ""
 
           - docker_image: zulip/ci:jammy
             name: Ubuntu 22.04 production install
             os: jammy
-            extra-args: ""
 
           - docker_image: zulip/ci:bullseye
             name: Debian 11 production install with custom db name and user
             os: bullseye
             extra-args: --test-custom-db
 
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install
-            os: bookworm
-            extra-args: ""
-
     name: ${{ matrix.name  }}
     container:
       image: ${{ matrix.docker_image }}
@@ -182,16 +160,25 @@ jobs:
           chmod +x /tmp/production-pgroonga
           chmod +x /tmp/production-install
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
+          chmod +x /tmp/send-failure-message
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
+      - name: Restore node_modules cache
+        uses: actions/cache@v3
+        with:
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('/tmp/package.json') }}-${{ hashFiles('/tmp/yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
+
       - name: Install production
-        run: sudo /tmp/production-install ${{ matrix.extra-args }}
+        run: |
+          sudo service rabbitmq-server restart
+          sudo /tmp/production-install ${{ matrix.extra-args }}
 
       - name: Verify install
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
@@ -212,22 +199,11 @@ jobs:
         if: ${{ matrix.os == 'focal' }}
         run: sudo /tmp/production-verify ${{ matrix.extra-args }}
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message
 
   production_upgrade:
     # The production upgrade job starts with a container with a
@@ -251,12 +227,6 @@ jobs:
           - docker_image: zulip/ci:bullseye-5.0
             name: 5.0 Version Upgrade
             os: bullseye
-          - docker_image: zulip/ci:bullseye-6.0
-            name: 6.0 Version Upgrade
-            os: bullseye
-          - docker_image: zulip/ci:bookworm-7.0
-            name: 7.0 Version Upgrade
-            os: bookworm
 
     name: ${{ matrix.name  }}
     container:
@@ -283,25 +253,14 @@ jobs:
           # of the tarball uploaded by the upload artifact fix those.
           chmod +x /tmp/production-upgrade
           chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
+          chmod +x /tmp/send-failure-message
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Temporarily bootstrap PostgreSQL upgrades
-        # https://chat.zulip.org/#narrow/stream/43-automated-testing/topic/postgres.20client.20upgrade.20failures/near/1640444
-        # On Debian, there is an ordering issue with post-install maintainer
-        # scripts when postgresql-client-common is upgraded at the same time as
-        # postgresql-client and postgresql-client-15.  Upgrade just
-        # postgresql-client-common first, so the main upgrade process can
-        # succeed.  This is a _temporary_ work-around to improve CI signal, as
-        # the failure does represent a real failure that production systems may
-        # encounter.
-        run: sudo apt-get update && sudo apt-get install -y --only-upgrade postgresql-client-common
-
       - name: Upgrade production
         run: sudo /tmp/production-upgrade
 
@@ -311,19 +270,8 @@ jobs:
         # - name: Verify install
         #   run: sudo /tmp/production-verify
 
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: /tmp/send-failure-message

.github/workflows/update-oneclick-apps.yml

@@ -22,6 +22,6 @@ jobs:
         run: |
           export PATH="$HOME/.local/bin:$PATH"
           git clone https://github.com/zulip/marketplace-partners
-          pip3 install python-digitalocean zulip fab-classic PyNaCl
+          pip3 install python-digitalocean zulip fab-classic
           echo $PATH
           python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

.github/workflows/zulip-ci.yml

@@ -27,6 +27,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        include_documentation_tests: [false]
+        include_frontend_tests: [false]
         include:
           # Base images are built using `tools/ci/Dockerfile.prod.template`.
           # The comments at the top explain how to build and upload these images.
@@ -34,26 +36,16 @@ jobs:
           - docker_image: zulip/ci:focal
             name: Ubuntu 20.04 (Python 3.8, backend + frontend)
             os: focal
-            include_documentation_tests: false
             include_frontend_tests: true
           # Debian 11 ships with Python 3.9.2.
           - docker_image: zulip/ci:bullseye
             name: Debian 11 (Python 3.9, backend + documentation)
             os: bullseye
             include_documentation_tests: true
-            include_frontend_tests: false
           # Ubuntu 22.04 ships with Python 3.10.4.
           - docker_image: zulip/ci:jammy
             name: Ubuntu 22.04 (Python 3.10, backend)
             os: jammy
-            include_documentation_tests: false
-            include_frontend_tests: false
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend)
-            os: bookworm
-            include_documentation_tests: false
-            include_frontend_tests: false
 
     runs-on: ubuntu-latest
     name: ${{ matrix.name }}
@@ -72,15 +64,16 @@ jobs:
 
       - name: Create cache directories
         run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
+          dirs=(/srv/zulip-{npm,venv,emoji}-cache)
           sudo mkdir -p "${dirs[@]}"
           sudo chown -R github "${dirs[@]}"
 
-      - name: Restore pnpm store
+      - name: Restore node_modules cache
         uses: actions/cache@v3
         with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+          path: /srv/zulip-npm-cache
+          key: v1-yarn-deps-${{ matrix.os }}-${{ hashFiles('package.json', 'yarn.lock') }}
+          restore-keys: v1-yarn-deps-${{ matrix.os }}
 
       - name: Restore python cache
         uses: actions/cache@v3
@@ -100,7 +93,10 @@ jobs:
         run: |
           # This is the main setup job for the test suite
           ./tools/ci/setup-backend --skip-dev-db-build
-          scripts/lib/clean_unused_caches.py --verbose --threshold=0
+
+          # Cleaning caches is mostly unnecessary in GitHub Actions, because
+          # most builds don't get to write to the cache.
+          # scripts/lib/clean_unused_caches.py --verbose --threshold 0
 
       - name: Run tools test
         run: |
@@ -112,26 +108,11 @@ jobs:
           source tools/ci/activate-venv
           ./tools/run-codespell
 
-      # We run the tests that are only run in a specific job early, so
-      # that we get feedback to the developer about likely failures as
-      # quickly as possible. Backend/mypy failures that aren't
-      # identical across different versions are much more rare than
-      # frontend linter or node test failures.
-      - name: Run documentation and api tests
-        if: ${{ matrix.include_documentation_tests }}
+      - name: Run backend lint
         run: |
           source tools/ci/activate-venv
-          # In CI, we only test links we control in test-documentation to avoid flakes
-          ./tools/test-documentation --skip-external-links
-          ./tools/test-help-documentation --skip-external-links
-          ./tools/test-api
-
-      - name: Run node tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Run the node tests first, since they're fast and deterministic
-          ./tools/test-js-with-node --coverage --parallel=1
+          echo "Test suite is running under $(python --version)."
+          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
 
       - name: Run frontend lint
         if: ${{ matrix.include_frontend_tests }}
@@ -139,41 +120,10 @@ jobs:
           source tools/ci/activate-venv
           ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
 
-      - name: Check schemas
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Check that various schemas are consistent. (is fast)
-          ./tools/check-schemas
-
-      - name: Check capitalization of strings
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./manage.py makemessages --locale en
-          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
-          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
-
-      - name: Run puppeteer tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-js-with-puppeteer
-
-      - name: Check pnpm dedupe
-        if: ${{ matrix.include_frontend_tests }}
-        run: pnpm dedupe --check
-
-      - name: Run backend lint
-        run: |
-          source tools/ci/activate-venv
-          echo "Test suite is running under $(python --version)."
-          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
-
       - name: Run backend tests
         run: |
           source tools/ci/activate-venv
-          ./tools/test-backend --coverage --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
+          ./tools/test-backend --coverage --include-webhooks --no-cov-cleanup --ban-console-output
 
       - name: Run mypy
         run: |
@@ -200,14 +150,50 @@ jobs:
           ./tools/test-migrations
           ./tools/setup/optimize-svg --check
           ./tools/setup/generate_integration_bots_avatars.py --check-missing
-          ./tools/ci/check-executables
 
-          # Ban check-database-compatibility from transitively
+          # Ban check-database-compatibility.py from transitively
           # relying on static/generated, because it might not be
           # up-to-date at that point in upgrade-zulip-stage-2.
-          chmod 000 static/generated web/generated
-          ./scripts/lib/check-database-compatibility
-          chmod 755 static/generated web/generated
+          chmod 000 static/generated
+          ./scripts/lib/check-database-compatibility.py
+          chmod 755 static/generated
+
+      - name: Run documentation and api tests
+        if: ${{ matrix.include_documentation_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # In CI, we only test links we control in test-documentation to avoid flakes
+          ./tools/test-documentation --skip-external-links
+          ./tools/test-help-documentation --skip-external-links
+          ./tools/test-api
+
+      - name: Run node tests
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # Run the node tests first, since they're fast and deterministic
+          ./tools/test-js-with-node --coverage --parallel=1
+
+      - name: Check schemas
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          # Check that various schemas are consistent. (is fast)
+          ./tools/check-schemas
+
+      - name: Check capitalization of strings
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          ./manage.py makemessages --locale en
+          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
+          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
+
+      - name: Run puppeteer tests
+        if: ${{ matrix.include_frontend_tests }}
+        run: |
+          source tools/ci/activate-venv
+          ./tools/test-js-with-puppeteer
 
       - name: Check for untracked files
         run: |
@@ -247,25 +233,8 @@ jobs:
       - name: Check development database build
         run: ./tools/ci/setup-backend
 
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
+      - name: Report status
+        if: failure()
+        env:
+          ZULIP_BOT_KEY: ${{ secrets.ZULIP_BOT_KEY }}
+        run: tools/ci/send-failure-message

.gitignore

@@ -17,10 +17,7 @@
 # See `git help ignore` for details on the format.
 
 ## Config files for the dev environment
-/zproject/apns-dev.pem
-/zproject/apns-dev-key.p8
 /zproject/dev-secrets.conf
-/zproject/custom_dev_settings.py
 /tools/conf.ini
 /tools/custom_provision
 /tools/droplets/conf.ini
@@ -36,7 +33,6 @@ package-lock.json
 !/var/puppeteer/test_credentials.d.ts
 
 /.dmypy.json
-/.ruff_cache
 
 # Generated i18n data
 /locale/en
@@ -47,11 +43,11 @@ package-lock.json
 # Static build
 *.mo
 npm-debug.log
-/.pnpm-store
 /node_modules
 /prod-static
 /staticfiles.json
 /webpack-stats-production.json
+/yarn-error.log
 zulip-git-version
 
 # Test / analysis tools
@@ -86,9 +82,6 @@ zulip.kdev4
 # Core dump files
 core
 
-# Static generated files for landing page.
-/static/images/landing-page/hello/generated
-
 ## Miscellaneous
 # (Ideally this section is empty.)
 .transifexrc

.gitlint

@@ -1,13 +1,13 @@
 [general]
 ignore=title-trailing-punctuation, body-min-length, body-is-missing
 
-extra-path=tools/lib/gitlint_rules.py
+extra-path=tools/lib/gitlint-rules.py
 
 [title-match-regex]
 regex=^(.+:\ )?[A-Z].+\.$
 
 [title-max-length]
-line-length=72
+line-length=76
 
 [body-max-line-length]
 line-length=76

.mailmap

@@ -12,129 +12,72 @@
 #     # shows raw names/emails, filtered by mapped name:
 #   $ git log --format='%an %ae' --author=$NAME | uniq -c
 
-acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
-acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
-Adam Benesh <Adam.Benesh@gmail.com>
 Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
-Adarsh Tiwari <xoldyckk@gmail.com>
+Adam Benesh <Adam.Benesh@gmail.com>
 Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
 Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
 Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
 Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
-Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
-Aman Agrawal <amanagr@zulip.com>
 Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
 Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
 Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Aryan Bhokare <aryan1bhokare@gmail.com>
-Aryan Bhokare <aryan1bhokare@gmail.com> <92683836+aryan-bhokare@users.noreply.github.com>
-Aryan Shridhar <aryanshridhar7@gmail.com>
 Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
+Aryan Shridhar <aryanshridhar7@gmail.com>
 Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
 Austin Riba <austin@zulip.com> <austin@m51.io>
 BIKI DAS <bikid475@gmail.com>
-Brijmohan Siyag <brijsiyag@gmail.com>
 Brock Whittaker <brock@zulipchat.com> <bjwhitta@asu.edu>
-Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Brock Whittaker <brock@zulipchat.com> <brockwhittaker@Brocks-MacBook.local>
+Brock Whittaker <brock@zulipchat.com> <brock@zulipchat.org>
 Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
 Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dinesh <chdinesh1089@gmail.com>
-Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
 Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
-Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
-Evy Kassirer <evy@zulip.com>
-Evy Kassirer <evy@zulip.com> <evy.kassirer@gmail.com>
-Evy Kassirer <evy@zulip.com> <evykassirer@users.noreply.github.com>
-Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
 Greg Price <greg@zulip.com> <gnprice@gmail.com>
 Greg Price <greg@zulip.com> <greg@zulipchat.com>
 Greg Price <greg@zulip.com> <price@mit.edu>
-Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
-Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
 Jai soni <jai_s@me.iitr.ac.in>
 Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
 Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
 Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-Joseph Ho <josephho678@gmail.com>
-Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
 Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
-Karl Stolley <karl@zulip.com> <karl@stolley.dev>
 Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
 Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
 Kevin Scott <kevin.scott.98@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com> <lalits01@smartek21.com>
-Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
 Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
 Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
-Matt Keller <matt@zulip.com>
-Matt Keller <matt@zulip.com> <m@cognusion.com>
-Nehal Sharma <bablinaneh@gmail.com>
-Nehal Sharma <bablinaneh@gmail.com> <68962290+N-Shar-ma@users.noreply.github.com>
-Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
-nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
-Palash Baderia <palash.baderia@outlook.com>
-Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
+m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
 Palash Raghuwanshi <singhpalash0@gmail.com>
 Parth <mittalparth22@gmail.com>
 Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
 Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
 Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
-Rein Zustand (rht) <rhtbot@protonmail.com>
-Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
 Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
 Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
-Rixant Rokaha <rixantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Rohan Gudimetla <rohan.gudimetla07@gmail.com>
-Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
-Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Satyam Bansal <sbansal1999@gmail.com>
+Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
 Sayam Samal <samal.sayam@gmail.com>
 Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
 Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
 Shlok Patel <shlokcpatel2001@gmail.com>
-Shu Chen <shu@zulip.com>
-Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
 Steve Howell <showell@zulip.com> <showell30@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@yahoo.com>
 Steve Howell <showell@zulip.com> <showell@zulipchat.com>
 Steve Howell <showell@zulip.com> <steve@humbughq.com>
 Steve Howell <showell@zulip.com> <steve@zulip.com>
 strifel <info@strifel.de>
-Tim Abbott <tabbott@zulip.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
 Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
 Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
-umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
-umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
-Viktor Illmer <1476338+v-ji@users.noreply.github.com>
 Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
 Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com>
+Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
+Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
 Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
+Yash RE <33805964+YashRE42@users.noreply.github.com>
 Yogesh Sirsat <yogeshsirsat56@gmail.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com>
 Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
-Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
-Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
-Zev Benjamin <zev@zulip.com> <zev@mit.edu>
-Zixuan James Li <p359101898@gmail.com>
-Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
-Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>
+Zeeshan Equbal <equbalzeeshan@gmail.com>

.npmrc

@@ -1 +0,0 @@
-ignore-dep-scripts=true

.prettierignore

@@ -1,11 +1,8 @@
-pnpm-lock.yaml
-/api_docs/**/*.md
 /corporate/tests/stripe_fixtures
-/help/**/*.md
 /locale
+/static/third
 /templates/**/*.md
 /tools/setup/emoji/emoji_map.json
-/web/third
 /zerver/tests/fixtures
 /zerver/webhooks/*/doc.md
 /zerver/webhooks/*/fixtures

.readthedocs.yaml

@@ -1,15 +0,0 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
-
-python:
-  install:
-    - requirements: requirements/docs.txt

.tx/config

@@ -1,39 +1,32 @@
-# Migrated from transifex-client format with `tx migrate`
-#
-# See https://developers.transifex.com/docs/using-the-client which hints at
-# this format, but in general, the headings are in the format of:
-#
-# [o:<org>:p:<project>:r:<resource>]
-
 [main]
 host = https://www.transifex.com
 lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
-[o:zulip:p:zulip:r:djangopo]
+[zulip.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip:r:mobile]
+[zulip.translationsjson]
+file_filter = locale/<lang>/translations.json
+source_file = locale/en/translations.json
+source_lang = en
+type = KEYVALUEJSON
+
+[zulip.mobile]
 file_filter = locale/<lang>/mobile.json
 source_file = locale/en/mobile.json
 source_lang = en
 type = KEYVALUEJSON
 
-[o:zulip:p:zulip:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip-test:r:djangopo]
+[zulip-test.djangopo]
 file_filter = locale/<lang>/LC_MESSAGES/django.po
 source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
 
-[o:zulip:p:zulip-test:r:translationsjson]
+[zulip-test.translationsjson]
 file_filter = locale/<lang>/translations.json
 source_file = locale/en/translations.json
 source_lang = en

.yarnrc

@@ -0,0 +1 @@
+ignore-scripts true

CODE_OF_CONDUCT.md

@@ -102,72 +102,3 @@ This Code of Conduct is adapted from the
 under a
 [Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
 license.
-
-## Moderating the Zulip community
-
-Anyone can help moderate the Zulip community by helping make sure that folks are
-aware of the [community guidelines](https://zulip.com/development-community/)
-and this Code of Conduct, and that we maintain a positive and respectful
-atmosphere.
-
-Here are some guidelines for you how can help:
-
-- Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
-  and just trying to keep the atmosphere warm make the whole community function
-  more smoothly. New participants who feel accepted, listened to and respected
-  are likely to treat others the same way.
-
-- Be familiar with the [community
-  guidelines](https://zulip.com/development-community/), and cite them liberally
-  when a user violates them. Be polite but firm. Some examples:
-
-  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
-    you ask a question. As noted in the [guidelines for this
-    community](https://zulip.com/development-community/):
-
-    > Use @-mentions sparingly… there is generally no need to @-mention a
-    > core contributor unless you need their timely attention.
-
-  - @user, please keep in mind the following [community
-    guideline](https://zulip.com/development-community/):
-
-    > Don’t ask the same question in multiple places. Moderators read every
-    > public stream, and make sure every question gets a reply.
-
-    I’ve gone ahead and moved the other copy of this message to this thread.
-
-  - If asked a question in a direct message that is better discussed in a public
-    stream:
-    > Hi @user! Please start by reviewing
-    > https://zulip.com/development-community/#community-norms to learn how to
-    > get help in this community.
-
-- Users sometimes think chat.zulip.org is a testing instance. When this happens,
-  kindly direct them to use the **#test here** stream.
-
-- If you see a message that’s posted in the wrong place, go ahead and move it if
-  you have permissions to do so, even if you don’t plan to respond to it.
-  Leaving the “Send automated notice to new topic” option enabled helps make it
-  clear what happened to the person who sent the message.
-
-  If you are responding to a message that's been moved, mention the user in your
-  reply, so that the mention serves as a notification of the new location for
-  their conversation.
-
-- If a user is posting spam, please report it to an administrator. They will:
-
-  - Change the user's name to `<name> (spammer)` and deactivate them.
-  - Delete any spam messages they posted in public streams.
-
-- We care very much about maintaining a respectful tone in our community. If you
-  see someone being mean or rude, point out that their tone is inappropriate,
-  and ask them to communicate their perspective in a respectful way in the
-  future. If you don’t feel comfortable doing so yourself, feel free to ask a
-  member of Zulip's core team to take care of the situation.
-
-- Try to assume the best intentions from others (given the range of
-  possibilities presented by their visible behavior), and stick with a friendly
-  and positive tone even when someone’s behavior is poor or disrespectful.
-  Everyone has bad days and stressful situations that can result in them
-  behaving not their best, and while we should be firm about our community
-  rules, we should also enforce them with kindness.

CONTRIBUTING.md

@@ -2,35 +2,16 @@
 
 Welcome to the Zulip community!
 
-## Zulip development community
+## Community
 
-The primary communication forum for the Zulip community is the Zulip
-server hosted at [chat.zulip.org](https://chat.zulip.org/):
-
-- **Users** and **administrators** of Zulip organizations stop by to
-  ask questions, offer feedback, and participate in product design
-  discussions.
-- **Contributors to the project**, including the **core Zulip
-  development team**, discuss ongoing and future projects, brainstorm
-  ideas, and generally help each other out.
-
-Everyone is welcome to [sign up](https://chat.zulip.org/) and
-participate — we love hearing from our users! Public streams in the
-community receive thousands of messages a week. We recommend signing
-up using the special invite links for
-[users](https://chat.zulip.org/join/t5crtoe62bpcxyisiyglmtvb/),
-[self-hosters](https://chat.zulip.org/join/wnhv3jzm6afa4raenedanfno/)
-and
-[contributors](https://chat.zulip.org/join/npzwak7vpmaknrhxthna3c7p/)
-to get a curated list of initial stream subscriptions.
-
-To learn how to get started participating in the community, including [community
-norms](https://zulip.com/development-community/#community-norms) and [where to
-post](https://zulip.com/development-community/#where-do-i-send-my-message),
-check out our [Zulip development community
-guide](https://zulip.com/development-community/). The Zulip community is
-governed by a [code of
-conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
+The
+[Zulip community server](https://zulip.com/development-community/)
+is the primary communication forum for the Zulip community. It is a good
+place to start whether you have a question, are a new contributor, are a new
+user, or anything else. Please review our
+[community norms](https://zulip.com/development-community/#community-norms)
+before posting. The Zulip community is also governed by a
+[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
 
 ## Ways to contribute
 
@@ -55,10 +36,8 @@ needs doing:
 **Non-code contributions**: Some of the most valuable ways to contribute
 don't require touching the codebase at all. For example, you can:
 
-- Report issues, including both [feature
-  requests](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html)
-  and [bug
-  reports](https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html).
+- [Report issues](#reporting-issues), including both feature requests and
+  bug reports.
 - [Give feedback](#user-feedback) if you are evaluating or using Zulip.
 - [Participate
   thoughtfully](https://zulip.readthedocs.io/en/latest/contributing/design-discussions.html)
@@ -78,13 +57,12 @@ to help.
 
 - First, make an account on the
   [Zulip community server](https://zulip.com/development-community/),
-  paying special attention to the
-  [community norms](https://zulip.com/development-community/#community-norms).
-  If you'd like, introduce yourself in
+  paying special attention to the community norms. If you'd like, introduce
+  yourself in
   [#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
   your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken or interesting/helpful as you
-  started using the product.
+  Zulip, and anything that felt confusing/broken as you started using the
+  product.
 - Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
 - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
   getting help in
@@ -149,15 +127,19 @@ Note that you are _not_ claiming an issue while you are iterating through steps
 1-4. _Before you claim an issue_, you should be confident that you will be able to
 tackle it effectively.
 
+If the lists of issues are overwhelming, you can post in
+[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
+bit about your background and interests, and we'll help you out. The most
+important thing to say is whether you're looking for a backend (Python),
+frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
+documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
+bit about your programming experience and available time.
+
 Additional tips for the [main server and web app
 repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
 
 - We especially recommend browsing recently opened issues, as there are more
   likely to be easy ones for you to find.
-- Take a look at issues with the ["good first issue"
-  label](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22),
-  as they are especially accessible to new contributors. However, you will
-  likely find issues without this label that are accessible as well.
 - All issues are partitioned into areas like
   admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
   through our [list of labels](https://github.com/zulip/zulip/labels), and
@@ -214,16 +196,101 @@ stream](https://chat.zulip.org/#narrow/stream/101-design) in the [Zulip
 development community](https://zulip.com/development-community/)
 
 For more advice, see [What makes a great Zulip
-contributor?](#what-makes-a-great-zulip-contributor) below. It's OK if your
-first issue takes you a while; that's normal! You'll be able to work a lot
-faster as you build experience.
+contributor?](#what-makes-a-great-zulip-contributor)
+below.
 
 ### Submitting a pull request
 
-See the [pull request review
-process](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-guide for detailed instructions on how to submit a pull request, and information
-on the stages of review your PR will go through.
+When you believe your code is ready, follow the [guide on how to review
+code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
+to review your own work. You can often find things you missed by taking a step
+back to look over your work before asking others to do so. Catching mistakes
+yourself will help your PRs be merged faster, and folks will appreciate the
+quality and professionalism of your work.
+
+Then, submit your changes. Carefully reading our [Git guide][git-guide], and in
+particular the section on [making a pull request][git-guide-make-pr], will help
+avoid many common mistakes. If any part of your contribution is from someone
+else (code snippets, images, sounds, or any other copyrightable work, modified
+or unmodified), be sure to review the instructions on how to [properly
+attribute][licensing] the work.
+
+[licensing]: https://zulip.readthedocs.io/en/latest/contributing/licensing.html#contributing-someone-else-s-work
+
+Once you are satisfied with the quality of your PR, follow the
+[guidelines on asking for a code
+review](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#asking-for-a-code-review)
+to request a review. If you are not sure what's best, simply post a
+comment on the main GitHub thread for your PR clearly indicating that
+it is ready for review, and the project maintainers will take a look
+and follow up with next steps.
+
+It's OK if your first issue takes you a while; that's normal! You'll be
+able to work a lot faster as you build experience.
+
+If it helps your workflow, you can submit your pull request marked as
+a [draft][github-help-draft-pr] while you're still working on it, and
+then mark it ready when you think it's time for someone else to review
+your work.
+
+[git-guide]: https://zulip.readthedocs.io/en/latest/git/
+[git-guide-make-pr]: https://zulip.readthedocs.io/en/latest/git/pull-requests.html
+[github-help-draft-pr]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests
+
+### Stages of a pull request
+
+Your pull request will likely go through several stages of review.
+
+1. If your PR makes user-facing changes, the UI and user experience may be
+   reviewed early on, without reference to the code. You will get feedback on
+   any user-facing bugs in the implementation. To minimize the number of review
+   round-trips, make sure to [thoroughly
+   test](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#manual-testing)
+   your own PR prior to asking for review.
+2. There may be choices made in the implementation that the reviewer
+   will ask you to revisit. This process will go more smoothly if you
+   specifically call attention to the decisions you made while
+   drafting the PR and any points about which you are uncertain. The
+   PR description and comments on your own PR are good ways to do this.
+3. Oftentimes, seeing an initial implementation will make it clear that the
+   product design for a feature needs to be revised, or that additional changes
+   are needed. The reviewer may therefore ask you to amend or change the
+   implementation. Some changes may be blockers for getting the PR merged, while
+   others may be improvements that can happen afterwards. Feel free to ask if
+   it's unclear which type of feedback you're getting. (Follow-ups can be a
+   great next issue to work on!)
+4. In addition to any UI/user experience review, all PRs will go through one or
+   more rounds of code review. Your code may initially be [reviewed by other
+   contributors](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html).
+   This helps us make good use of project maintainers' time, and helps you make
+   progress on the PR by getting more frequent feedback. A project maintainer
+   may leave a comment asking someone with expertise in the area you're working
+   on to review your work.
+5. Final code review and integration for server and web app PRs is generally done
+   by `@timabbott`.
+
+#### How to help move the review process forward
+
+The key to keeping your review moving through the review process is to:
+
+- Address _all_ the feedback to the best of your ability.
+- Make it clear when the requested changes have been made
+  and you believe it's time for another look.
+- Make it as easy as possible to review the changes you made.
+
+In order to do this, when you believe you have addressed the previous round of
+feedback on your PR as best you can, post a comment asking reviewers to take
+another look. Your comment should make it easy to understand what has been done
+and what remains by:
+
+- Summarizing the changes made since the last review you received.
+- Highlighting remaining questions or decisions, with links to any relevant
+  chat.zulip.org threads.
+- Providing updated screenshots and information on manual testing if
+  appropriate.
+
+The easier it is to review your work, the more likely you are to receive quick
+feedback.
 
 ### Beyond the first issue
 
@@ -249,34 +316,17 @@ labels.
   use the existing pull request (PR) as a starting point for your contribution. If
   you think a different approach is needed, you can post a new PR, with a comment that clearly
   explains _why_ you decided to start from scratch.
-- **What if I ask if someone is still working on an issue, and they don't
-  respond?** If you don't get a reply within 2-3 days, go ahead and post a comment
-  that you are working on the issue, and submit a pull request. If the original
-  assignee ends up submitting a pull request first, no worries! You can help by
-  providing feedback on their work, or submit your own PR if you think a
-  different approach is needed (as described above).
 - **Can I come up with my own feature idea and work on it?** We welcome
   suggestions of features or other improvements that you feel would be valuable. If you
   have a new feature you'd like to add, you can start a conversation [in our
   development community](https://zulip.com/development-community/#where-do-i-send-my-message)
   explaining the feature idea and the problem that you're hoping to solve.
-- **I'm waiting for the next round of review on my PR. Can I pick up
-  another issue in the meantime?** Someone's first Zulip PR often
-  requires quite a bit of iteration, so please [make sure your pull
-  request is reviewable][reviewable-pull-requests] and go through at
-  least one round of feedback from others before picking up a second
-  issue. After that, sure! If
-  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
-  claim an issue, you can post a comment describing the status of your
-  other work on the issue you're interested in, and asking for the
-  issue to be assigned to you. Note that addressing feedback on
-  in-progress PRs should always take priority over starting a new PR.
 - **I think my PR is done, but it hasn't been merged yet. What's going on?**
   1. **Double-check that you have addressed all the feedback**, including any comments
      on [Git commit
-     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
+     discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
   2. If all the feedback has been addressed, did you [leave a
-     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
+     comment](#how-to-help-move-the-review-process-forward)
      explaining that you have done so and **requesting another review**? If not,
      it may not be clear to project maintainers or reviewers that your PR is
      ready for another look.
@@ -292,8 +342,6 @@ labels.
      occasionally take a few weeks for a PR in the final stages of the review
      process to be merged.
 
-[reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
-
 ## What makes a great Zulip contributor?
 
 Zulip has a lot of experience working with new contributors. In our
@@ -305,7 +353,7 @@ experience, these are the best predictors of success:
   you got stuck. Post tracebacks or other error messages if appropriate. For
   more advice, check out [our guide][great-questions]!
 - Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
+  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
 - Submitting carefully tested code. See our [detailed guide on how to review
   code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
   (yours or someone else's).
@@ -326,6 +374,29 @@ experience, these are the best predictors of success:
 
 [great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
 
+## Reporting issues
+
+If you find an easily reproducible bug and/or are experienced in reporting
+bugs, feel free to just open an issue on the relevant project on GitHub.
+
+If you have a feature request or are not yet sure what the underlying bug
+is, the best place to post issues is
+[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
+[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
+[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
+[Zulip community server](https://zulip.com/development-community/).
+This allows us to interactively figure out what is going on, let you know if
+a similar issue has already been opened, and collect any other information
+we need. Choose a 2-4 word topic that describes the issue, explain the issue
+and how to reproduce it if known, your browser/OS if relevant, and a
+[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
+if appropriate.
+
+**Reporting security issues**. Please do not report security issues
+publicly, including on public streams on chat.zulip.org. You can
+email [security@zulip.com](mailto:security@zulip.com). We create a CVE for every
+security issue in our released software.
+
 ## User feedback
 
 Nearly every feature we develop starts with a user request. If you are part

Dockerfile-postgresql

@@ -8,7 +8,7 @@
 #       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
 
 # Currently the PostgreSQL images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image cannot currently be upgraded
+# the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
 
 # https://hub.docker.com/r/groonga/pgroonga/tags

README.md

@@ -17,7 +17,6 @@ Come find us on the [development community chat](https://zulip.com/development-c
 [![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
 [![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
 [![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
 [![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)

SECURITY.md

@@ -33,5 +33,5 @@ See also our documentation on the [Zulip release
 lifecycle][release-lifecycle].
 
 [security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
-[upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
+[upgrades]: https://zulip.readthedocs.io/en/latest/production/upgrade-or-modify.html#upgrading-to-a-release
 [release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html

analytics/lib/counts.py

@@ -8,7 +8,6 @@ from django.conf import settings
 from django.db import connection, models
 from django.db.models import F
 from psycopg2.sql import SQL, Composable, Identifier, Literal
-from typing_extensions import TypeAlias, override
 
 from analytics.models import (
     BaseCount,
@@ -19,20 +18,14 @@ from analytics.models import (
     UserCount,
     installation_epoch,
 )
+from zerver.lib.logging_util import log_to_file
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC
 from zerver.models import Message, Realm, RealmAuditLog, Stream, UserActivityInterval, UserProfile
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import (
-        RemoteInstallationCount,
-        RemoteRealm,
-        RemoteRealmCount,
-        RemoteZulipServer,
-    )
-
-
-logger = logging.getLogger("zulip.analytics")
+## Logging setup ##
 
+logger = logging.getLogger("zulip.management")
+log_to_file(logger, settings.ANALYTICS_LOG_PATH)
 
 # You can't subtract timedelta.max from a datetime, so use this instead
 TIMEDELTA_MAX = timedelta(days=365 * 1000)
@@ -69,8 +62,7 @@ class CountStat:
         else:
             self.interval = self.time_increment
 
-    @override
-    def __repr__(self) -> str:
+    def __str__(self) -> str:
         return f"<CountStat: {self.property}>"
 
     def last_successful_fill(self) -> Optional[datetime]:
@@ -109,9 +101,6 @@ class DataCollector:
         self.output_table = output_table
         self.pull_function = pull_function
 
-    def depends_on_realm(self) -> bool:
-        return self.output_table in (UserCount, StreamCount)
-
 
 ## CountStat-level operations ##
 
@@ -200,7 +189,7 @@ def do_fill_count_stat_at_hour(
 def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
     if isinstance(stat, LoggingCountStat):
         InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
-        if stat.data_collector.depends_on_realm():
+        if stat.data_collector.output_table in [UserCount, StreamCount]:
             RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
     else:
         UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
@@ -221,7 +210,7 @@ def do_aggregate_to_summary_table(
     else:
         realm_clause = SQL("")
 
-    if stat.data_collector.depends_on_realm():
+    if output_table in (UserCount, StreamCount):
         realmcount_query = SQL(
             """
             INSERT INTO analytics_realmcount
@@ -299,10 +288,9 @@ def do_aggregate_to_summary_table(
 
 ## Utility functions called from outside counts.py ##
 
-
 # called from zerver.actions; should not throw any errors
 def do_increment_logging_stat(
-    model_object_for_bucket: Union[Realm, UserProfile, Stream, "RemoteRealm", "RemoteZulipServer"],
+    zerver_object: Union[Realm, UserProfile, Stream],
     stat: CountStat,
     subgroup: Optional[Union[str, int, bool]],
     event_time: datetime,
@@ -313,37 +301,21 @@ def do_increment_logging_stat(
 
     table = stat.data_collector.output_table
     if table == RealmCount:
-        assert isinstance(model_object_for_bucket, Realm)
-        id_args: Dict[
-            str, Optional[Union[Realm, UserProfile, Stream, "RemoteRealm", "RemoteZulipServer"]]
-        ] = {"realm": model_object_for_bucket}
+        assert isinstance(zerver_object, Realm)
+        id_args: Dict[str, Union[Realm, UserProfile, Stream]] = {"realm": zerver_object}
     elif table == UserCount:
-        assert isinstance(model_object_for_bucket, UserProfile)
-        id_args = {"realm": model_object_for_bucket.realm, "user": model_object_for_bucket}
-    elif table == StreamCount:
-        assert isinstance(model_object_for_bucket, Stream)
-        id_args = {"realm": model_object_for_bucket.realm, "stream": model_object_for_bucket}
-    elif table == RemoteInstallationCount:
-        assert isinstance(model_object_for_bucket, RemoteZulipServer)
-        id_args = {"server": model_object_for_bucket, "remote_id": None}
-    elif table == RemoteRealmCount:
-        assert isinstance(model_object_for_bucket, RemoteRealm)
-        id_args = {
-            "server": model_object_for_bucket.server,
-            "remote_realm": model_object_for_bucket,
-            "remote_id": None,
-        }
-    else:
-        raise AssertionError("Unsupported CountStat output_table")
+        assert isinstance(zerver_object, UserProfile)
+        id_args = {"realm": zerver_object.realm, "user": zerver_object}
+    else:  # StreamCount
+        assert isinstance(zerver_object, Stream)
+        id_args = {"realm": zerver_object.realm, "stream": zerver_object}
 
     if stat.frequency == CountStat.DAY:
         end_time = ceiling_to_day(event_time)
-    elif stat.frequency == CountStat.HOUR:
+    else:  # CountStat.HOUR:
         end_time = ceiling_to_hour(event_time)
-    else:
-        raise AssertionError("Unsupported CountStat frequency")
 
-    row, created = table._default_manager.get_or_create(
+    row, created = table.objects.get_or_create(
         property=stat.property,
         subgroup=subgroup,
         end_time=end_time,
@@ -373,7 +345,7 @@ def do_drop_single_stat(property: str) -> None:
 
 ## DataCollector-level operations ##
 
-QueryFn: TypeAlias = Callable[[Dict[str, Composable]], Composable]
+QueryFn = Callable[[Dict[str, Composable]], Composable]
 
 
 def do_pull_by_sql_query(
@@ -473,13 +445,7 @@ def count_message_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -506,13 +472,7 @@ def count_message_type_by_user_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        # We limit both userprofile and message so that we only see
-        # users from this realm, but also get the performance speedup
-        # of limiting messages by realm.
-        realm_clause = SQL(
-            "zerver_userprofile.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_userprofile.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_usercount
@@ -561,10 +521,7 @@ def count_message_by_stream_query(realm: Optional[Realm]) -> QueryFn:
     if realm is None:
         realm_clause: Composable = SQL("")
     else:
-        realm_clause = SQL(
-            "zerver_stream.realm_id = {} AND zerver_message.realm_id = {} AND"
-        ).format(Literal(realm.id), Literal(realm.id))
-    # Uses index: zerver_message_realm_date_sent (or the only-date index)
+        realm_clause = SQL("zerver_stream.realm_id = {} AND").format(Literal(realm.id))
     return lambda kwargs: SQL(
         """
     INSERT INTO analytics_streamcount
@@ -842,12 +799,6 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
         CountStat(
             "minutes_active::day", DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY
         ),
-        # Tracks the number of push notifications requested by the server.
-        LoggingCountStat(
-            "mobile_pushes_sent::day",
-            RealmCount,
-            CountStat.DAY,
-        ),
         # Rate limiting stats
         # Used to limit the number of invitation emails sent by a realm
         LoggingCountStat("invites_sent::day", RealmCount, CountStat.DAY),
@@ -862,65 +813,8 @@ def get_count_stats(realm: Optional[Realm] = None) -> Dict[str, CountStat]:
         ),
     ]
 
-    if settings.ZILENCER_ENABLED:
-        # See also the remote_installation versions of these in REMOTE_INSTALLATION_COUNT_STATS.
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_received::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-        count_stats_.append(
-            LoggingCountStat(
-                "mobile_pushes_forwarded::day",
-                RemoteRealmCount,
-                CountStat.DAY,
-            )
-        )
-
     return OrderedDict((stat.property, stat) for stat in count_stats_)
 
 
-# These properties are tracked by the bouncer itself and therefore syncing them
-# from a remote server should not be allowed - or the server would be able to interfere
-# with our data.
-BOUNCER_ONLY_REMOTE_COUNT_STAT_PROPERTIES = [
-    "mobile_pushes_received::day",
-    "mobile_pushes_forwarded::day",
-]
-
 # To avoid refactoring for now COUNT_STATS can be used as before
 COUNT_STATS = get_count_stats()
-
-REMOTE_INSTALLATION_COUNT_STATS = OrderedDict()
-
-if settings.ZILENCER_ENABLED:
-    # REMOTE_INSTALLATION_COUNT_STATS contains duplicates of the
-    # RemoteRealmCount stats declared above; it is necessary because
-    # pre-8.0 servers do not send the fields required to identify a
-    # RemoteRealm.
-
-    # Tracks the number of push notifications requested to be sent
-    # by a remote server.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_received::day"] = LoggingCountStat(
-        "mobile_pushes_received::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-    # Tracks the number of push notifications successfully sent to
-    # mobile devices, as requested by the remote server. Therefore
-    # this should be less than or equal to mobile_pushes_received -
-    # with potential tiny offsets resulting from a request being
-    # *received* by the bouncer right before midnight, but *sent* to
-    # the mobile device right after midnight. This would cause the
-    # increments to happen to CountStat records for different days.
-    REMOTE_INSTALLATION_COUNT_STATS["mobile_pushes_forwarded::day"] = LoggingCountStat(
-        "mobile_pushes_forwarded::day",
-        RemoteInstallationCount,
-        CountStat.DAY,
-    )
-
-ALL_COUNT_STATS = OrderedDict(
-    list(COUNT_STATS.items()) + list(REMOTE_INSTALLATION_COUNT_STATS.items())
-)

analytics/lib/fixtures.py

@@ -1,5 +1,5 @@
 from math import sqrt
-from random import Random
+from random import gauss, random, seed
 from typing import List
 
 from analytics.lib.counts import CountStat
@@ -36,8 +36,6 @@ def generate_time_series_data(
     partial_sum -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
-    rng = Random(random_seed)
-
     if frequency == CountStat.HOUR:
         length = days * 24
         seasonality = [non_business_hours_base] * 24 * 7
@@ -46,13 +44,13 @@ def generate_time_series_data(
                 seasonality[24 * day + hour] = business_hours_base
         holidays = []
         for i in range(days):
-            holidays.extend([rng.random() < holiday_rate] * 24)
+            holidays.extend([random() < holiday_rate] * 24)
     elif frequency == CountStat.DAY:
         length = days
         seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
             24 * non_business_hours_base
         ] * 2
-        holidays = [rng.random() < holiday_rate for i in range(days)]
+        holidays = [random() < holiday_rate for i in range(days)]
     else:
         raise AssertionError(f"Unknown frequency: {frequency}")
     if length < 2:
@@ -64,10 +62,11 @@ def generate_time_series_data(
         seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)
     ]
 
-    noise_scalars = [rng.gauss(0, 1)]
+    seed(random_seed)
+    noise_scalars = [gauss(0, 1)]
     for i in range(1, length):
         noise_scalars.append(
-            noise_scalars[-1] * autocorrelation + rng.gauss(0, 1) * (1 - autocorrelation)
+            noise_scalars[-1] * autocorrelation + gauss(0, 1) * (1 - autocorrelation)
         )
 
     values = [

analytics/lib/time_utils.py

@@ -30,5 +30,4 @@ def time_range(
     while current >= start:
         times.append(current)
         current -= step
-    times.reverse()
-    return times
+    return list(reversed(times))

analytics/management/commands/check_analytics_state.py

@@ -5,11 +5,10 @@ from typing import Any, Dict
 
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, CountStat
+from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.models import installation_epoch
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
+from zerver.lib.timestamp import TimeZoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC
 from zerver.models import Realm
 
 states = {
@@ -25,7 +24,6 @@ class Command(BaseCommand):
 
     Run as a cron job that runs every hour."""
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         fill_state = self.get_fill_state()
         status = fill_state["status"]
@@ -44,13 +42,13 @@ class Command(BaseCommand):
 
         warning_unfilled_properties = []
         critical_unfilled_properties = []
-        for property, stat in ALL_COUNT_STATS.items():
+        for property, stat in COUNT_STATS.items():
             last_fill = stat.last_successful_fill()
             if last_fill is None:
                 last_fill = installation_epoch()
             try:
                 verify_UTC(last_fill)
-            except TimeZoneNotUTCError:
+            except TimeZoneNotUTCException:
                 return {"status": 2, "message": f"FillState not in UTC for {property}"}
 
             if stat.frequency == CountStat.DAY:

analytics/management/commands/clear_analytics_tables.py

@@ -2,7 +2,6 @@ from argparse import ArgumentParser
 from typing import Any
 
 from django.core.management.base import BaseCommand, CommandError
-from typing_extensions import override
 
 from analytics.lib.counts import do_drop_all_analytics_tables
 
@@ -10,11 +9,9 @@ from analytics.lib.counts import do_drop_all_analytics_tables
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         if options["force"]:
             do_drop_all_analytics_tables()

analytics/management/commands/clear_single_stat.py

@@ -2,23 +2,20 @@ from argparse import ArgumentParser
 from typing import Any
 
 from django.core.management.base import BaseCommand, CommandError
-from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
+from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
 
 
 class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument("--force", action="store_true", help="Actually do it.")
         parser.add_argument("--property", help="The property of the stat to be cleared.")
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         property = options["property"]
-        if property not in ALL_COUNT_STATS:
+        if property not in COUNT_STATS:
             raise CommandError(f"Invalid property: {property}")
         if not options["force"]:
             raise CommandError("No action taken. Use --force.")

analytics/management/commands/populate_analytics_db.py

@@ -5,7 +5,6 @@ from typing import Any, Dict, List, Mapping, Type, Union
 from django.core.files.uploadedfile import UploadedFile
 from django.core.management.base import BaseCommand
 from django.utils.timezone import now as timezone_now
-from typing_extensions import TypeAlias, override
 
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
@@ -24,18 +23,8 @@ from zerver.lib.create_user import create_user
 from zerver.lib.storage import static_path
 from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
-from zerver.lib.upload import upload_message_attachment_from_request
-from zerver.models import (
-    Client,
-    Realm,
-    RealmAuditLog,
-    Recipient,
-    Stream,
-    Subscription,
-    SystemGroups,
-    UserGroup,
-    UserProfile,
-)
+from zerver.lib.upload import upload_message_image_from_request
+from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserGroup, UserProfile
 
 
 class Command(BaseCommand):
@@ -69,7 +58,6 @@ class Command(BaseCommand):
             random_seed=self.random_seed,
         )
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         # TODO: This should arguably only delete the objects
         # associated with the "analytics" realm.
@@ -104,18 +92,8 @@ class Command(BaseCommand):
         )
         do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None)
 
-        # Create guest user for set_guest_users_statistic.
-        create_user(
-            "bassanio@analytics.ds",
-            "Bassanio",
-            realm,
-            full_name="Bassanio",
-            role=UserProfile.ROLE_GUEST,
-            force_date_joined=installation_time,
-        )
-
         administrators_user_group = UserGroup.objects.get(
-            name=SystemGroups.ADMINISTRATORS, realm=realm, is_system_group=True
+            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=realm, is_system_group=True
         )
         stream = Stream.objects.create(
             name="all",
@@ -128,29 +106,25 @@ class Command(BaseCommand):
         stream.save(update_fields=["recipient"])
 
         # Subscribe shylock to the stream to avoid invariant failures.
-        Subscription.objects.create(
-            recipient=recipient,
-            user_profile=shylock,
-            is_user_active=shylock.is_active,
-            color=STREAM_ASSIGNMENT_COLORS[0],
-        )
-        RealmAuditLog.objects.create(
-            realm=realm,
-            modified_user=shylock,
-            modified_stream=stream,
-            event_last_message_id=0,
-            event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
-            event_time=installation_time,
-        )
+        # TODO: This should use subscribe_users_to_streams from populate_db.
+        subs = [
+            Subscription(
+                recipient=recipient,
+                user_profile=shylock,
+                is_user_active=shylock.is_active,
+                color=STREAM_ASSIGNMENT_COLORS[0],
+            ),
+        ]
+        Subscription.objects.bulk_create(subs)
 
         # Create an attachment in the database for set_storage_space_used_statistic.
         IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
         file_info = os.stat(IMAGE_FILE_PATH)
         file_size = file_info.st_size
         with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock, file_size)
+            upload_message_image_from_request(UploadedFile(fp), shylock, file_size)
 
-        FixtureData: TypeAlias = Mapping[Union[str, int, None], List[int]]
+        FixtureData = Mapping[Union[str, int, None], List[int]]
 
         def insert_fixture_data(
             stat: CountStat,
@@ -158,7 +132,7 @@ class Command(BaseCommand):
             table: Type[BaseCount],
         ) -> None:
             end_times = time_range(
-                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
+                last_end_time, last_end_time, stat.frequency, len(list(fixture_data.values())[0])
             )
             if table == InstallationCount:
                 id_args: Dict[str, Any] = {}
@@ -170,7 +144,7 @@ class Command(BaseCommand):
                 id_args = {"stream": stream, "realm": realm}
 
             for subgroup, values in fixture_data.items():
-                table._default_manager.bulk_create(
+                table.objects.bulk_create(
                     table(
                         property=stat.property,
                         subgroup=subgroup,

analytics/management/commands/update_analytics_counts.py

@@ -1,4 +1,3 @@
-import hashlib
 import os
 import time
 from argparse import ArgumentParser
@@ -9,11 +8,10 @@ from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.dateparse import parse_datetime
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
-from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
 from scripts.lib.zulip_tools import ENDC, WARNING
-from zerver.lib.remote_server import send_server_data_to_push_bouncer
+from zerver.lib.remote_server import send_analytics_to_remote_server
 from zerver.lib.timestamp import floor_to_hour
 from zerver.models import Realm
 
@@ -23,7 +21,6 @@ class Command(BaseCommand):
 
     Run as a cron job that runs every hour."""
 
-    @override
     def add_arguments(self, parser: ArgumentParser) -> None:
         parser.add_argument(
             "--time",
@@ -40,7 +37,6 @@ class Command(BaseCommand):
             "--verbose", action="store_true", help="Print timing information to stdout."
         )
 
-    @override
     def handle(self, *args: Any, **options: Any) -> None:
         try:
             os.mkdir(settings.ANALYTICS_LOCK_DIR)
@@ -75,9 +71,9 @@ class Command(BaseCommand):
         fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
 
         if options["stat"] is not None:
-            stats = [ALL_COUNT_STATS[options["stat"]]]
+            stats = [COUNT_STATS[options["stat"]]]
         else:
-            stats = list(ALL_COUNT_STATS.values())
+            stats = list(COUNT_STATS.values())
 
         logger.info("Starting updating analytics counts through %s", fill_to_time)
         if options["verbose"]:
@@ -96,14 +92,5 @@ class Command(BaseCommand):
             )
         logger.info("Finished updating analytics counts through %s", fill_to_time)
 
-        if settings.PUSH_NOTIFICATION_BOUNCER_URL:
-            # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
-            # that each server will report in at a somewhat consistent time.
-            assert settings.ZULIP_ORG_ID
-            delay = int.from_bytes(
-                hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
-            ) % (60 * 10)
-            logger.info("Sleeping %d seconds before reporting...", delay)
-            time.sleep(delay)
-
-            send_server_data_to_push_bouncer(consider_usage_statistics=True)
+        if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
+            send_analytics_to_remote_server()

analytics/migrations/0001_initial.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0030_realm_org_type"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),

analytics/migrations/0002_remove_huddlecount.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0001_initial"),
     ]

analytics/migrations/0003_fillstate.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0002_remove_huddlecount"),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0003_fillstate"),
     ]

analytics/migrations/0005_alter_field_size.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0004_add_subgroup"),
     ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -2,6 +2,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0005_alter_field_size"),
     ]

analytics/migrations/0007_remove_interval.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0006_add_subgroup_to_unique_constraints"),
     ]

analytics/migrations/0008_add_count_indexes.py

@@ -1,33 +1,25 @@
 # Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations, models
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0050_userprofile_avatar_version"),
         ("analytics", "0007_remove_interval"),
     ]
 
     operations = [
-        migrations.AddIndex(
-            model_name="realmcount",
-            index=models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="realmcount",
+            index_together={("property", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="streamcount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="streamcount",
+            index_together={("property", "realm", "end_time")},
         ),
-        migrations.AddIndex(
-            model_name="usercount",
-            index=models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            ),
+        migrations.AlterIndexTogether(
+            name="usercount",
+            index_together={("property", "realm", "end_time")},
         ),
     ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -21,6 +21,7 @@ def delete_messages_sent_to_stream_stat(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0008_add_count_indexes"),
     ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -21,6 +21,7 @@ def clear_message_sent_by_message_type_values(
 
 
 class Migration(migrations.Migration):
+
     dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
 
     operations = [

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 
@@ -18,6 +18,7 @@ def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0010_clear_messages_sent_values"),
     ]

analytics/migrations/0012_add_on_delete.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0011_clear_analytics_tables"),
     ]

analytics/migrations/0013_remove_anomaly.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0012_add_on_delete"),
     ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0013_remove_anomaly"),
     ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,5 +1,5 @@
 from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 from django.db.models import Count, Sum
 
@@ -55,6 +55,7 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEdi
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0014_remove_fillstate_last_modified"),
     ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("analytics", "0015_clear_duplicate_counts"),
     ]

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -1,114 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0016_unique_constraint_when_subgroup_null"),
-    ]
-
-    # If the server was installed between 7.0 and 7.4 (or main between
-    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
-    # running 7.5 or 7807bff52635 or higher) are never used, because
-    # they contain an improper cast
-    # (https://code.djangoproject.com/ticket/34840).
-    #
-    # We regenerate the indexes here, by dropping and re-creating
-    # them, so that we know that they are properly formed.
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/models.py

@@ -1,11 +1,7 @@
-# https://github.com/typeddjango/django-stubs/issues/1698
-# mypy: disable-error-code="explicit-override"
-
-from datetime import datetime
+import datetime
 
 from django.db import models
 from django.db.models import Q, UniqueConstraint
-from typing_extensions import override
 
 from zerver.lib.timestamp import floor_to_day
 from zerver.models import Realm, Stream, UserProfile
@@ -20,14 +16,13 @@ class FillState(models.Model):
     STARTED = 2
     state = models.PositiveSmallIntegerField()
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.end_time} {self.state}"
+        return f"<FillState: {self.property} {self.end_time} {self.state}>"
 
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime:
+def installation_epoch() -> datetime.datetime:
     earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
         "date_created__min"
     ]
@@ -63,9 +58,8 @@ class InstallationCount(BaseCount):
             ),
         ]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.property} {self.subgroup} {self.value}"
+        return f"<InstallationCount: {self.property} {self.subgroup} {self.value}>"
 
 
 class RealmCount(BaseCount):
@@ -85,16 +79,10 @@ class RealmCount(BaseCount):
                 name="unique_realm_count_null_subgroup",
             ),
         ]
-        indexes = [
-            models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            )
-        ]
+        index_together = ["property", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
+        return f"<RealmCount: {self.realm} {self.property} {self.subgroup} {self.value}>"
 
 
 class UserCount(BaseCount):
@@ -117,16 +105,10 @@ class UserCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from users to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
+        return f"<UserCount: {self.user} {self.property} {self.subgroup} {self.value}>"
 
 
 class StreamCount(BaseCount):
@@ -149,13 +131,9 @@ class StreamCount(BaseCount):
         ]
         # This index dramatically improves the performance of
         # aggregating from streams to realms
-        indexes = [
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            )
-        ]
+        index_together = ["property", "realm", "end_time"]
 
-    @override
     def __str__(self) -> str:
-        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"
+        return (
+            f"<StreamCount: {self.stream} {self.property} {self.subgroup} {self.value} {self.id}>"
+        )

analytics/tests/test_activity_views.py

@@ -1,80 +1,10 @@
-import uuid
-from datetime import timedelta
 from unittest import mock
 
 from django.utils.timezone import now as timezone_now
 
-from corporate.lib.stripe import add_months
-from corporate.models import Customer, CustomerPlan, LicenseLedger
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.models import Client, UserActivity, UserProfile
-from zilencer.models import (
-    RemoteRealmAuditLog,
-    RemoteZulipServer,
-    get_remote_server_guest_and_non_guest_count,
-)
-
-event_time = timezone_now() - timedelta(days=3)
-data_list = [
-    {
-        "server_id": 1,
-        "realm_id": 1,
-        "event_type": RemoteRealmAuditLog.USER_CREATED,
-        "event_time": event_time,
-        "extra_data": {
-            RemoteRealmAuditLog.ROLE_COUNT: {
-                RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
-                    UserProfile.ROLE_REALM_ADMINISTRATOR: 10,
-                    UserProfile.ROLE_REALM_OWNER: 10,
-                    UserProfile.ROLE_MODERATOR: 10,
-                    UserProfile.ROLE_MEMBER: 10,
-                    UserProfile.ROLE_GUEST: 10,
-                }
-            }
-        },
-    },
-    {
-        "server_id": 1,
-        "realm_id": 1,
-        "event_type": RemoteRealmAuditLog.USER_ROLE_CHANGED,
-        "event_time": event_time,
-        "extra_data": {
-            RemoteRealmAuditLog.ROLE_COUNT: {
-                RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
-                    UserProfile.ROLE_REALM_ADMINISTRATOR: 20,
-                    UserProfile.ROLE_REALM_OWNER: 0,
-                    UserProfile.ROLE_MODERATOR: 0,
-                    UserProfile.ROLE_MEMBER: 20,
-                    UserProfile.ROLE_GUEST: 10,
-                }
-            }
-        },
-    },
-    {
-        "server_id": 1,
-        "realm_id": 2,
-        "event_type": RemoteRealmAuditLog.USER_CREATED,
-        "event_time": event_time,
-        "extra_data": {
-            RemoteRealmAuditLog.ROLE_COUNT: {
-                RemoteRealmAuditLog.ROLE_COUNT_HUMANS: {
-                    UserProfile.ROLE_REALM_ADMINISTRATOR: 10,
-                    UserProfile.ROLE_REALM_OWNER: 10,
-                    UserProfile.ROLE_MODERATOR: 0,
-                    UserProfile.ROLE_MEMBER: 10,
-                    UserProfile.ROLE_GUEST: 5,
-                }
-            }
-        },
-    },
-    {
-        "server_id": 1,
-        "realm_id": 2,
-        "event_type": RemoteRealmAuditLog.USER_CREATED,
-        "event_time": event_time,
-        "extra_data": {},
-    },
-]
+from zerver.lib.test_helpers import queries_captured
+from zerver.models import Client, UserActivity, UserProfile, flush_per_request_caches
 
 
 class ActivityTest(ZulipTestCase):
@@ -102,57 +32,24 @@ class ActivityTest(ZulipTestCase):
         user_profile.is_staff = True
         user_profile.save(update_fields=["is_staff"])
 
-        with self.assert_database_query_count(11):
+        flush_per_request_caches()
+        with queries_captured() as queries:
             result = self.client_get("/activity")
             self.assertEqual(result.status_code, 200)
 
-        # Add data for remote activity page
-        RemoteRealmAuditLog.objects.bulk_create([RemoteRealmAuditLog(**data) for data in data_list])
-        remote_server = RemoteZulipServer.objects.get(id=1)
-        customer = Customer.objects.create(remote_server=remote_server)
-        plan = CustomerPlan.objects.create(
-            customer=customer,
-            billing_cycle_anchor=timezone_now(),
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            tier=CustomerPlan.TIER_SELF_HOSTED_BUSINESS,
-            price_per_license=8000,
-            next_invoice_date=add_months(timezone_now(), 12),
-        )
-        LicenseLedger.objects.create(
-            licenses=10,
-            licenses_at_next_renewal=10,
-            event_time=timezone_now(),
-            is_renewal=True,
-            plan=plan,
-        )
-        RemoteZulipServer.objects.create(
-            uuid=str(uuid.uuid4()),
-            api_key="magic_secret_api_key",
-            hostname="demo.example.com",
-            contact_email="email@example.com",
-        )
-        with self.assert_database_query_count(10):
-            result = self.client_get("/activity/remote")
-            self.assertEqual(result.status_code, 200)
+        self.assert_length(queries, 19)
 
-        with self.assert_database_query_count(4):
-            result = self.client_get("/activity/integrations")
-            self.assertEqual(result.status_code, 200)
-
-        with self.assert_database_query_count(8):
+        flush_per_request_caches()
+        with queries_captured() as queries:
             result = self.client_get("/realm_activity/zulip/")
             self.assertEqual(result.status_code, 200)
 
+        self.assert_length(queries, 8)
+
         iago = self.example_user("iago")
-        with self.assert_database_query_count(5):
+        flush_per_request_caches()
+        with queries_captured() as queries:
             result = self.client_get(f"/user_activity/{iago.id}/")
             self.assertEqual(result.status_code, 200)
 
-    def test_get_remote_server_guest_and_non_guest_count(self) -> None:
-        RemoteRealmAuditLog.objects.bulk_create([RemoteRealmAuditLog(**data) for data in data_list])
-
-        remote_server_counts = get_remote_server_guest_and_non_guest_count(
-            server_id=1, event_time=timezone_now()
-        )
-        self.assertEqual(remote_server_counts.non_guest_user_count, 70)
-        self.assertEqual(remote_server_counts.guest_user_count, 15)
+        self.assert_length(queries, 5)

analytics/tests/test_counts.py

@@ -3,14 +3,11 @@ from typing import Any, Dict, List, Optional, Tuple, Type
 from unittest import mock
 
 import orjson
-import time_machine
 from django.apps import apps
 from django.db import models
 from django.db.models import Sum
-from django.test import override_settings
 from django.utils.timezone import now as timezone_now
 from psycopg2.sql import SQL, Literal
-from typing_extensions import override
 
 from analytics.lib.counts import (
     COUNT_STATS,
@@ -55,26 +52,19 @@ from zerver.actions.user_activity import update_user_activity_interval
 from zerver.actions.users import do_deactivate_user
 from zerver.lib.create_user import create_user
 from zerver.lib.exceptions import InvitationError
-from zerver.lib.push_notifications import (
-    get_message_payload_apns,
-    get_message_payload_gcm,
-    hex_to_b64,
-)
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import TimeZoneNotUTCError, ceiling_to_day, floor_to_day
+from zerver.lib.timestamp import TimeZoneNotUTCException, floor_to_day
 from zerver.lib.topic import DB_TOPIC_NAME
 from zerver.lib.utils import assert_is_not_none
 from zerver.models import (
     Client,
     Huddle,
     Message,
-    NotificationTriggers,
     PreregistrationUser,
     Realm,
     RealmAuditLog,
     Recipient,
     Stream,
-    SystemGroups,
     UserActivityInterval,
     UserGroup,
     UserProfile,
@@ -82,14 +72,6 @@ from zerver.models import (
     get_user,
     is_cross_realm_bot_email,
 )
-from zilencer.models import (
-    RemoteInstallationCount,
-    RemotePushDeviceToken,
-    RemoteRealm,
-    RemoteRealmCount,
-    RemoteZulipServer,
-)
-from zilencer.views import get_last_id_from_server
 
 
 class AnalyticsTestCase(ZulipTestCase):
@@ -99,16 +81,13 @@ class AnalyticsTestCase(ZulipTestCase):
     TIME_ZERO = datetime(1988, 3, 14, tzinfo=timezone.utc)
     TIME_LAST_HOUR = TIME_ZERO - HOUR
 
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.default_realm = do_create_realm(
             string_id="realmtest", name="Realm Test", date_created=self.TIME_ZERO - 2 * self.DAY
         )
         self.administrators_user_group = UserGroup.objects.get(
-            name=SystemGroups.ADMINISTRATORS,
-            realm=self.default_realm,
-            is_system_group=True,
+            name=UserGroup.ADMINISTRATORS_GROUP_NAME, realm=self.default_realm, is_system_group=True
         )
 
         # used to generate unique names in self.create_*
@@ -116,10 +95,6 @@ class AnalyticsTestCase(ZulipTestCase):
         # used as defaults in self.assert_table_count
         self.current_property: Optional[str] = None
 
-        # Delete RemoteRealm registrations to have a clean slate - the relevant
-        # tests want to construct this from scratch.
-        RemoteRealm.objects.all().delete()
-
     # Lightweight creation of users, streams, and messages
     def create_user(self, **kwargs: Any) -> UserProfile:
         self.name_counter += 1
@@ -134,7 +109,7 @@ class AnalyticsTestCase(ZulipTestCase):
         for key, value in defaults.items():
             kwargs[key] = kwargs.get(key, value)
         kwargs["delivery_email"] = kwargs["email"]
-        with time_machine.travel(kwargs["date_joined"], tick=False):
+        with mock.patch("zerver.lib.create_user.timezone_now", return_value=kwargs["date_joined"]):
             pass_kwargs: Dict[str, Any] = {}
             if kwargs["is_bot"]:
                 pass_kwargs["bot_type"] = UserProfile.DEFAULT_BOT
@@ -207,9 +182,7 @@ class AnalyticsTestCase(ZulipTestCase):
     ) -> None:
         if property is None:
             property = self.current_property
-        queryset = table._default_manager.filter(property=property, end_time=end_time).filter(
-            **kwargs
-        )
+        queryset = table.objects.filter(property=property, end_time=end_time).filter(**kwargs)
         if table is not InstallationCount:
             if realm is None:
                 realm = self.default_realm
@@ -254,18 +227,16 @@ class AnalyticsTestCase(ZulipTestCase):
                 kwargs[arg_keys[i]] = values[i]
             for key, value in defaults.items():
                 kwargs[key] = kwargs.get(key, value)
-            if (
-                table not in [InstallationCount, RemoteInstallationCount, RemoteRealmCount]
-                and "realm" not in kwargs
-            ):
-                if "user" in kwargs:
-                    kwargs["realm"] = kwargs["user"].realm
-                elif "stream" in kwargs:
-                    kwargs["realm"] = kwargs["stream"].realm
-                else:
-                    kwargs["realm"] = self.default_realm
-            self.assertEqual(table._default_manager.filter(**kwargs).count(), 1)
-        self.assert_length(arg_values, table._default_manager.count())
+            if table is not InstallationCount:
+                if "realm" not in kwargs:
+                    if "user" in kwargs:
+                        kwargs["realm"] = kwargs["user"].realm
+                    elif "stream" in kwargs:
+                        kwargs["realm"] = kwargs["stream"].realm
+                    else:
+                        kwargs["realm"] = self.default_realm
+            self.assertEqual(table.objects.filter(**kwargs).count(), 1)
+        self.assert_length(arg_values, table.objects.count())
 
 
 class TestProcessCountStat(AnalyticsTestCase):
@@ -319,7 +290,7 @@ class TestProcessCountStat(AnalyticsTestCase):
         stat = self.make_dummy_count_stat("test stat")
         with self.assertRaises(ValueError):
             process_count_stat(stat, installation_epoch() + 65 * self.MINUTE)
-        with self.assertRaises(TimeZoneNotUTCError):
+        with self.assertRaises(TimeZoneNotUTCException):
             process_count_stat(stat, installation_epoch().replace(tzinfo=None))
 
     # This tests the LoggingCountStat branch of the code in do_delete_counts_at_hour.
@@ -483,7 +454,6 @@ class TestProcessCountStat(AnalyticsTestCase):
 
 
 class TestCountStats(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         # This tests two things for each of the queries/CountStats: Handling
@@ -688,7 +658,7 @@ class TestCountStats(AnalyticsTestCase):
         self.create_message(user1, recipient_huddle1)
         self.create_message(user2, recipient_huddle2)
 
-        # direct messages
+        # private messages
         recipient_user1 = Recipient.objects.get(type_id=user1.id, type=Recipient.PERSONAL)
         recipient_user2 = Recipient.objects.get(type_id=user2.id, type=Recipient.PERSONAL)
         recipient_user3 = Recipient.objects.get(type_id=user3.id, type=Recipient.PERSONAL)
@@ -1398,252 +1368,6 @@ class TestLoggingCountStats(AnalyticsTestCase):
             ],
         )
 
-    @override_settings(PUSH_NOTIFICATION_BOUNCER_URL="https://push.zulip.org.example.com")
-    def test_mobile_pushes_received_count(self) -> None:
-        self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe"
-        self.server = RemoteZulipServer.objects.create(
-            uuid=self.server_uuid,
-            api_key="magic_secret_api_key",
-            hostname="demo.example.com",
-            last_updated=timezone_now(),
-        )
-
-        hamlet = self.example_user("hamlet")
-        token = "aaaa"
-
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.GCM,
-            token=hex_to_b64(token),
-            user_uuid=(hamlet.uuid),
-            server=self.server,
-        )
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.GCM,
-            token=hex_to_b64(token + "aa"),
-            user_uuid=(hamlet.uuid),
-            server=self.server,
-        )
-        RemotePushDeviceToken.objects.create(
-            kind=RemotePushDeviceToken.APNS,
-            token=hex_to_b64(token),
-            user_uuid=str(hamlet.uuid),
-            server=self.server,
-        )
-
-        message = Message(
-            sender=hamlet,
-            recipient=self.example_user("othello").recipient,
-            realm_id=hamlet.realm_id,
-            content="This is test content",
-            rendered_content="This is test content",
-            date_sent=timezone_now(),
-            sending_client=get_client("test"),
-        )
-        message.set_topic_name("Test topic")
-        message.save()
-        gcm_payload, gcm_options = get_message_payload_gcm(hamlet, message)
-        apns_payload = get_message_payload_apns(
-            hamlet, message, NotificationTriggers.DIRECT_MESSAGE
-        )
-
-        # First we'll make a request without providing realm_uuid. That means
-        # the bouncer can't increment the RemoteRealmCount stat, and only
-        # RemoteInstallationCount will be incremented.
-        payload = {
-            "user_id": hamlet.id,
-            "user_uuid": str(hamlet.uuid),
-            "gcm_payload": gcm_payload,
-            "apns_payload": apns_payload,
-            "gcm_options": gcm_options,
-        }
-        now = timezone_now()
-        with time_machine.travel(now, tick=False), mock.patch(
-            "zilencer.views.send_android_push_notification", return_value=1
-        ), mock.patch("zilencer.views.send_apple_push_notification", return_value=1), mock.patch(
-            "corporate.lib.stripe.RemoteServerBillingSession.current_count_for_billed_licenses",
-            return_value=10,
-        ), self.assertLogs(
-            "zilencer.views", level="INFO"
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # There are 3 devices we created for the user:
-        # 1. The mobile_pushes_received increment should match that number.
-        # 2. mobile_pushes_forwarded only counts successful deliveries, and we've set up
-        #    the mocks above to simulate 1 successful android and 1 successful apple delivery.
-        #    Thus the increment should be just 2.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    3,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    2,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-            ],
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_received::day").exists()
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_forwarded::day").exists()
-        )
-
-        # Now provide the realm_uuid. However, the RemoteRealm record doesn't exist yet, so it'll
-        # still be ignored.
-        payload = {
-            "user_id": hamlet.id,
-            "user_uuid": str(hamlet.uuid),
-            "realm_uuid": str(hamlet.realm.uuid),
-            "gcm_payload": gcm_payload,
-            "apns_payload": apns_payload,
-            "gcm_options": gcm_options,
-        }
-        with time_machine.travel(now, tick=False), mock.patch(
-            "zilencer.views.send_android_push_notification", return_value=1
-        ), mock.patch("zilencer.views.send_apple_push_notification", return_value=1), mock.patch(
-            "corporate.lib.stripe.RemoteServerBillingSession.current_count_for_billed_licenses",
-            return_value=10,
-        ), self.assertLogs(
-            "zilencer.views", level="INFO"
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # The RemoteInstallationCount records get incremented again, but the RemoteRealmCount
-        # remains ignored due to missing RemoteRealm record.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    6,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    4,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-            ],
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_received::day").exists()
-        )
-        self.assertFalse(
-            RemoteRealmCount.objects.filter(property="mobile_pushes_forwarded::day").exists()
-        )
-
-        # Create the RemoteRealm registration and repeat the above. This time RemoteRealmCount
-        # stats should be collected.
-        realm = hamlet.realm
-        remote_realm = RemoteRealm.objects.create(
-            server=self.server,
-            uuid=realm.uuid,
-            uuid_owner_secret=realm.uuid_owner_secret,
-            host=realm.host,
-            realm_deactivated=realm.deactivated,
-            realm_date_created=realm.date_created,
-        )
-
-        with time_machine.travel(now, tick=False), mock.patch(
-            "zilencer.views.send_android_push_notification", return_value=1
-        ), mock.patch("zilencer.views.send_apple_push_notification", return_value=1), mock.patch(
-            "corporate.lib.stripe.RemoteServerBillingSession.current_count_for_billed_licenses",
-            return_value=10,
-        ), self.assertLogs(
-            "zilencer.views", level="INFO"
-        ):
-            result = self.uuid_post(
-                self.server_uuid,
-                "/api/v1/remotes/push/notify",
-                payload,
-                content_type="application/json",
-                subdomain="",
-            )
-            self.assert_json_success(result)
-
-        # The RemoteInstallationCount records get incremented again, and the RemoteRealmCount
-        # gets collected.
-        self.assertTableState(
-            RemoteInstallationCount,
-            ["property", "value", "subgroup", "server", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    9,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    6,
-                    None,
-                    self.server,
-                    None,
-                    ceiling_to_day(now),
-                ],
-            ],
-        )
-        self.assertTableState(
-            RemoteRealmCount,
-            ["property", "value", "subgroup", "server", "remote_realm", "remote_id", "end_time"],
-            [
-                [
-                    "mobile_pushes_received::day",
-                    3,
-                    None,
-                    self.server,
-                    remote_realm,
-                    None,
-                    ceiling_to_day(now),
-                ],
-                [
-                    "mobile_pushes_forwarded::day",
-                    2,
-                    None,
-                    self.server,
-                    remote_realm,
-                    None,
-                    ceiling_to_day(now),
-                ],
-            ],
-        )
-
     def test_invites_sent(self) -> None:
         property = "invites_sent::day"
 
@@ -1659,48 +1383,46 @@ class TestLoggingCountStats(AnalyticsTestCase):
         stream, _ = self.create_stream_with_recipient()
 
         invite_expires_in_minutes = 2 * 24 * 60
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_invite_users(
-                user,
-                ["user1@domain.tld", "user2@domain.tld"],
-                [stream],
-                invite_expires_in_minutes=invite_expires_in_minutes,
-            )
+        do_invite_users(
+            user,
+            ["user1@domain.tld", "user2@domain.tld"],
+            [stream],
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
         assertInviteCountEquals(2)
 
         # We currently send emails when re-inviting users that haven't
         # turned into accounts, so count them towards the total
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_invite_users(
-                user,
-                ["user1@domain.tld", "user2@domain.tld"],
-                [stream],
-                invite_expires_in_minutes=invite_expires_in_minutes,
-            )
+        do_invite_users(
+            user,
+            ["user1@domain.tld", "user2@domain.tld"],
+            [stream],
+            invite_expires_in_minutes=invite_expires_in_minutes,
+        )
         assertInviteCountEquals(4)
 
         # Test mix of good and malformed invite emails
-        with self.assertRaises(InvitationError), mock.patch(
-            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
-        ):
+        try:
             do_invite_users(
                 user,
                 ["user3@domain.tld", "malformed"],
                 [stream],
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
+        except InvitationError:
+            pass
         assertInviteCountEquals(4)
 
         # Test inviting existing users
-        with self.assertRaises(InvitationError), mock.patch(
-            "zerver.actions.invites.too_many_recent_realm_invites", return_value=False
-        ):
+        try:
             do_invite_users(
                 user,
                 ["first@domain.tld", "user4@domain.tld"],
                 [stream],
                 invite_expires_in_minutes=invite_expires_in_minutes,
             )
+        except InvitationError:
+            pass
         assertInviteCountEquals(5)
 
         # Revoking invite should not give you credit
@@ -1710,8 +1432,7 @@ class TestLoggingCountStats(AnalyticsTestCase):
         assertInviteCountEquals(5)
 
         # Resending invite should cost you
-        with mock.patch("zerver.actions.invites.too_many_recent_realm_invites", return_value=False):
-            do_resend_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
+        do_resend_user_invite_email(assert_is_not_none(PreregistrationUser.objects.first()))
         assertInviteCountEquals(6)
 
     def test_messages_read_hour(self) -> None:
@@ -1784,12 +1505,12 @@ class TestDeleteStats(AnalyticsTestCase):
         FillState.objects.create(property="test", end_time=self.TIME_ZERO, state=FillState.DONE)
 
         analytics = apps.get_app_config("analytics")
-        for table in analytics.models.values():
-            self.assertTrue(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertTrue(table.objects.exists())
 
         do_drop_all_analytics_tables()
-        for table in analytics.models.values():
-            self.assertFalse(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertFalse(table.objects.exists())
 
     def test_do_drop_single_stat(self) -> None:
         user = self.create_user()
@@ -1808,17 +1529,16 @@ class TestDeleteStats(AnalyticsTestCase):
         FillState.objects.create(property="to_save", end_time=self.TIME_ZERO, state=FillState.DONE)
 
         analytics = apps.get_app_config("analytics")
-        for table in analytics.models.values():
-            self.assertTrue(table._default_manager.exists())
+        for table in list(analytics.models.values()):
+            self.assertTrue(table.objects.exists())
 
         do_drop_single_stat("to_delete")
-        for table in analytics.models.values():
-            self.assertFalse(table._default_manager.filter(property="to_delete").exists())
-            self.assertTrue(table._default_manager.filter(property="to_save").exists())
+        for table in list(analytics.models.values()):
+            self.assertFalse(table.objects.filter(property="to_delete").exists())
+            self.assertTrue(table.objects.filter(property="to_save").exists())
 
 
 class TestActiveUsersAudit(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.user = self.create_user()
@@ -2001,7 +1721,6 @@ class TestActiveUsersAudit(AnalyticsTestCase):
 
 
 class TestRealmActiveHumans(AnalyticsTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.stat = COUNT_STATS["realm_active_humans::day"]
@@ -2121,26 +1840,3 @@ class TestRealmActiveHumans(AnalyticsTestCase):
             1,
         )
         self.assertEqual(RealmCount.objects.filter(property="realm_active_humans::day").count(), 1)
-
-
-class GetLastIdFromServerTest(ZulipTestCase):
-    def test_get_last_id_from_server_ignores_null(self) -> None:
-        """
-        Verifies that get_last_id_from_server ignores null remote_ids, since this goes
-        against the default Postgres ordering behavior, which treats nulls as the largest value.
-        """
-        self.server_uuid = "6cde5f7a-1f7e-4978-9716-49f69ebfc9fe"
-        self.server = RemoteZulipServer.objects.create(
-            uuid=self.server_uuid,
-            api_key="magic_secret_api_key",
-            hostname="demo.example.com",
-            last_updated=timezone_now(),
-        )
-        first = RemoteInstallationCount.objects.create(
-            end_time=timezone_now(), server=self.server, property="test", value=1, remote_id=1
-        )
-        RemoteInstallationCount.objects.create(
-            end_time=timezone_now(), server=self.server, property="test2", value=1, remote_id=None
-        )
-        result = get_last_id_from_server(self.server, RemoteInstallationCount)
-        self.assertEqual(result, first.remote_id)

analytics/tests/test_stats_views.py

@@ -2,11 +2,10 @@ from datetime import datetime, timedelta, timezone
 from typing import List, Optional
 
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, StreamCount, UserCount
+from analytics.models import FillState, RealmCount, UserCount
 from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
@@ -69,12 +68,10 @@ class TestStatsEndpoint(ZulipTestCase):
 
 
 class TestGetChartData(ZulipTestCase):
-    @override
     def setUp(self) -> None:
         super().setUp()
         self.realm = get_realm("zulip")
         self.user = self.example_user("hamlet")
-        self.stream_id = self.get_stream_id(self.get_streams(self.user)[0])
         self.login_user(self.user)
         self.end_times_hour = [
             ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
@@ -117,17 +114,6 @@ class TestGetChartData(ZulipTestCase):
             )
             for i, subgroup in enumerate(user_subgroups)
         )
-        StreamCount.objects.bulk_create(
-            StreamCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                stream_id=self.stream_id,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
         FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
 
     def test_number_of_humans(self) -> None:
@@ -193,20 +179,20 @@ class TestGetChartData(ZulipTestCase):
                 "everyone": {
                     "Public streams": self.data(100),
                     "Private streams": self.data(0),
-                    "Direct messages": self.data(101),
-                    "Group direct messages": self.data(0),
+                    "Private messages": self.data(101),
+                    "Group private messages": self.data(0),
                 },
                 "user": {
                     "Public streams": self.data(200),
                     "Private streams": self.data(201),
-                    "Direct messages": self.data(0),
-                    "Group direct messages": self.data(0),
+                    "Private messages": self.data(0),
+                    "Group private messages": self.data(0),
                 },
                 "display_order": [
-                    "Direct messages",
+                    "Private messages",
                     "Public streams",
                     "Private streams",
-                    "Group direct messages",
+                    "Group private messages",
                 ],
                 "result": "success",
             },
@@ -264,49 +250,6 @@ class TestGetChartData(ZulipTestCase):
             },
         )
 
-    def test_messages_sent_by_stream(self) -> None:
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
-        self.insert_data(stat, ["true", "false"], [])
-
-        result = self.client_get(
-            f"/json/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-        result = self.api_get(
-            self.example_user("polonius"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        self.assert_json_error(result, "Not allowed for guest users")
-
-        # Verify we correctly forbid access to stats of streams in other realms.
-        result = self.api_get(
-            self.mit_user("sipbtest"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-            subdomain="zephyr",
-        )
-        self.assert_json_error(result, "Invalid stream ID")
-
     def test_include_empty_subgroups(self) -> None:
         FillState.objects.create(
             property="realm_active_humans::day",
@@ -344,8 +287,8 @@ class TestGetChartData(ZulipTestCase):
             {
                 "Public streams": [0],
                 "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
+                "Private messages": [0],
+                "Group private messages": [0],
             },
         )
         self.assertEqual(
@@ -353,8 +296,8 @@ class TestGetChartData(ZulipTestCase):
             {
                 "Public streams": [0],
                 "Private streams": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
+                "Private messages": [0],
+                "Group private messages": [0],
             },
         )
 

analytics/tests/test_support_views.py

@@ -1,31 +1,22 @@
 from datetime import datetime, timedelta, timezone
-from decimal import Decimal
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Optional
 from unittest import mock
 
 import orjson
-import time_machine
 from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
-from corporate.lib.stripe import RealmBillingSession, add_months
-from corporate.models import (
-    Customer,
-    CustomerPlan,
-    LicenseLedger,
-    SponsoredPlanTypes,
-    ZulipSponsorshipRequest,
-    get_current_plan_by_realm,
-    get_customer_by_realm,
-)
+from corporate.lib.stripe import add_months, update_sponsorship_status
+from corporate.models import Customer, CustomerPlan, LicenseLedger, get_customer_by_realm
 from zerver.actions.invites import do_create_multiuse_invite_link
-from zerver.actions.realm_settings import do_change_realm_org_type, do_send_realm_reactivation_email
-from zerver.actions.user_settings import do_change_user_setting
+from zerver.actions.realm_settings import (
+    do_change_realm_org_type,
+    do_send_realm_reactivation_email,
+    do_set_realm_property,
+)
 from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm
+from zerver.lib.test_helpers import reset_emails_in_zulip_realm
 from zerver.models import (
     MultiuseInvite,
-    OrgTypeEnum,
     PreregistrationUser,
     Realm,
     UserMessage,
@@ -33,153 +24,14 @@ from zerver.models import (
     get_org_type_display_name,
     get_realm,
 )
-from zilencer.lib.remote_counts import MissingDataError
 
 if TYPE_CHECKING:
     from django.test.client import _MonkeyPatchedWSGIResponse as TestHttpResponse
 
-import uuid
-
-from zilencer.models import RemoteZulipServer
-
-
-class TestRemoteServerSupportEndpoint(ZulipTestCase):
-    @override
-    def setUp(self) -> None:
-        def add_sponsorship_request(
-            hostname: str, org_type: int, website: str, paid_users: str, plan: str
-        ) -> None:
-            remote_server = RemoteZulipServer.objects.get(hostname=hostname)
-            customer = Customer.objects.create(
-                remote_server=remote_server, sponsorship_pending=True
-            )
-            ZulipSponsorshipRequest.objects.create(
-                customer=customer,
-                org_type=org_type,
-                org_website=website,
-                org_description="We help people.",
-                expected_total_users="20-35",
-                paid_users_count=paid_users,
-                paid_users_description="",
-                requested_plan=plan,
-            )
-
-        super().setUp()
-
-        # Set up some initial example data.
-        for i in range(20):
-            hostname = f"zulip-{i}.example.com"
-            RemoteZulipServer.objects.create(
-                hostname=hostname, contact_email=f"admin@{hostname}", plan_type=1, uuid=uuid.uuid4()
-            )
-
-        # Add example sponsorship request data
-        add_sponsorship_request(
-            hostname="zulip-1.example.com",
-            org_type=OrgTypeEnum.Community.value,
-            website="",
-            paid_users="None",
-            plan=SponsoredPlanTypes.BUSINESS.value,
-        )
-
-        add_sponsorship_request(
-            hostname="zulip-2.example.com",
-            org_type=OrgTypeEnum.OpenSource.value,
-            website="example.org",
-            paid_users="",
-            plan=SponsoredPlanTypes.COMMUNITY.value,
-        )
-
-    def test_search(self) -> None:
-        self.login("cordelia")
-
-        result = self.client_get("/activity/remote/support")
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        # Iago is the user with the appropriate permissions to access this page.
-        self.login("iago")
-        assert self.example_user("iago").is_staff
-
-        result = self.client_get("/activity/remote/support")
-        self.assert_in_success_response(
-            [
-                'input type="text" name="q" class="input-xxlarge search-query" placeholder="hostname or contact email"'
-            ],
-            result,
-        )
-
-        with mock.patch("analytics.views.support.compute_max_monthly_messages", return_value=1000):
-            result = self.client_get("/activity/remote/support", {"q": "zulip-1.example.com"})
-        self.assert_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-        self.assert_in_success_response(["<b>Max monthly messages</b>: 1000"], result)
-        self.assert_not_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
-
-        # Sponsorship request information
-        self.assert_in_success_response(["<li><b>Organization type</b>: Community</li>"], result)
-        self.assert_in_success_response(
-            ["<li><b>Organization website</b>: No website submitted</li>"], result
-        )
-        self.assert_in_success_response(["<li><b>Paid users</b>: None</li>"], result)
-        self.assert_in_success_response(["<li><b>Requested plan</b>: Business</li>"], result)
-        self.assert_in_success_response(
-            ["<li><b>Organization description</b>: We help people.</li>"], result
-        )
-        self.assert_in_success_response(["<li><b>Estimated total users</b>: 20-35</li>"], result)
-        self.assert_in_success_response(["<li><b>Description of paid users</b>: </li>"], result)
-
-        with mock.patch(
-            "analytics.views.support.compute_max_monthly_messages", side_effect=MissingDataError
-        ):
-            result = self.client_get("/activity/remote/support", {"q": "zulip-1.example.com"})
-        self.assert_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-        self.assert_in_success_response(
-            ["<b>Max monthly messages</b>: Recent data missing"], result
-        )
-        self.assert_not_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
-
-        result = self.client_get("/activity/remote/support", {"q": "example.com"})
-        for i in range(20):
-            self.assert_in_success_response([f"<h3>zulip-{i}.example.com</h3>"], result)
-
-        result = self.client_get("/activity/remote/support", {"q": "admin@zulip-2.example.com"})
-        self.assert_in_success_response(["<h3>zulip-2.example.com</h3>"], result)
-        self.assert_in_success_response(["<b>Contact email</b>: admin@zulip-2.example.com"], result)
-        self.assert_not_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-
-        # Sponsorship request information
-        self.assert_in_success_response(
-            ["<li><b>Organization type</b>: Open-source project</li>"], result
-        )
-        self.assert_in_success_response(
-            ["<li><b>Organization website</b>: example.org</li>"], result
-        )
-        self.assert_in_success_response(["<li><b>Paid users</b>: </li>"], result)
-        self.assert_in_success_response(["<li><b>Requested plan</b>: Community</li>"], result)
-        self.assert_in_success_response(
-            ["<li><b>Organization description</b>: We help people.</li>"], result
-        )
-        self.assert_in_success_response(["<li><b>Estimated total users</b>: 20-35</li>"], result)
-        self.assert_in_success_response(["<li><b>Description of paid users</b>: </li>"], result)
-
-        result = self.client_get("/activity/remote/support", {"q": "admin@zulip-3.example.com"})
-        self.assert_in_success_response(["<h3>zulip-3.example.com</h3>"], result)
-        self.assert_in_success_response(["<b>Contact email</b>: admin@zulip-3.example.com"], result)
-        self.assert_not_in_success_response(["<h3>zulip-1.example.com</h3>"], result)
-
-        # Sponsorship request information
-        self.assert_not_in_success_response(
-            ["<li><b>Organization description</b>: We help people.</li>"], result
-        )
-        self.assert_not_in_success_response(
-            ["<li><b>Estimated total users</b>: 20-35</li>"], result
-        )
-        self.assert_not_in_success_response(["<li><b>Description of paid users</b>: </li>"], result)
-
 
 class TestSupportEndpoint(ZulipTestCase):
     def test_search(self) -> None:
-        reset_email_visibility_to_everyone_in_zulip_realm()
+        reset_emails_in_zulip_realm()
         lear_user = self.lear_user("king")
         lear_user.is_staff = True
         lear_user.save(update_fields=["is_staff"])
@@ -278,7 +130,7 @@ class TestSupportEndpoint(ZulipTestCase):
                     '<option value="deactivated" >Deactivated</option>',
                     'scrub-realm-button">',
                     'data-string-id="lear"',
-                    "<b>Plan name</b>: Zulip Cloud Standard",
+                    "<b>Name</b>: Zulip Cloud Standard",
                     "<b>Status</b>: Active",
                     "<b>Billing schedule</b>: Annual",
                     "<b>Licenses</b>: 2/10 (Manual)",
@@ -366,10 +218,10 @@ class TestSupportEndpoint(ZulipTestCase):
 
         self.login("iago")
 
-        do_change_user_setting(
-            self.example_user("hamlet"),
+        do_set_realm_property(
+            get_realm("zulip"),
             "email_address_visibility",
-            UserProfile.EMAIL_ADDRESS_VISIBILITY_NOBODY,
+            Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY,
             acting_user=None,
         )
 
@@ -378,8 +230,8 @@ class TestSupportEndpoint(ZulipTestCase):
         plan = CustomerPlan.objects.create(
             customer=customer,
             billing_cycle_anchor=now,
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            tier=CustomerPlan.TIER_CLOUD_STANDARD,
+            billing_schedule=CustomerPlan.ANNUAL,
+            tier=CustomerPlan.STANDARD,
             price_per_license=8000,
             next_invoice_date=add_months(now, 12),
         )
@@ -400,12 +252,6 @@ class TestSupportEndpoint(ZulipTestCase):
         check_hamlet_user_query_result(result)
         check_zulip_realm_query_result(result)
 
-        # Search should be case-insensitive:
-        assert self.example_email("hamlet") != self.example_email("hamlet").upper()
-        result = get_check_query_result(self.example_email("hamlet").upper(), 1)
-        check_hamlet_user_query_result(result)
-        check_zulip_realm_query_result(result)
-
         result = get_check_query_result(lear_user.email, 1)
         check_lear_user_query_result(result)
         check_lear_realm_query_result(result)
@@ -443,56 +289,53 @@ class TestSupportEndpoint(ZulipTestCase):
         check_zulip_realm_query_result(result)
         check_lear_realm_query_result(result)
 
-        self.client_post("/accounts/home/", {"email": self.nonreg_email("test")})
-        self.login("iago")
+        with mock.patch(
+            "analytics.views.support.timezone_now",
+            return_value=timezone_now() - timedelta(minutes=50),
+        ):
+            self.client_post("/accounts/home/", {"email": self.nonreg_email("test")})
+            self.login("iago")
+            result = get_check_query_result(self.nonreg_email("test"), 1)
+            check_preregistration_user_query_result(result, self.nonreg_email("test"))
+            check_zulip_realm_query_result(result)
 
-        def query_result_from_before(*args: Any) -> "TestHttpResponse":
-            with time_machine.travel((timezone_now() - timedelta(minutes=50)), tick=False):
-                return get_check_query_result(*args)
+            create_invitation("Denmark", self.nonreg_email("test1"))
+            result = get_check_query_result(self.nonreg_email("test1"), 1)
+            check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
+            check_zulip_realm_query_result(result)
 
-        result = query_result_from_before(self.nonreg_email("test"), 1)
-        check_preregistration_user_query_result(result, self.nonreg_email("test"))
-        check_zulip_realm_query_result(result)
+            email = self.nonreg_email("alice")
+            self.client_post("/new/", {"email": email})
+            result = get_check_query_result(email, 1)
+            check_realm_creation_query_result(result, email)
 
-        create_invitation("Denmark", self.nonreg_email("test1"))
-        result = query_result_from_before(self.nonreg_email("test1"), 1)
-        check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
-        check_zulip_realm_query_result(result)
+            invite_expires_in_minutes = 10 * 24 * 60
+            do_create_multiuse_invite_link(
+                self.example_user("hamlet"),
+                invited_as=1,
+                invite_expires_in_minutes=invite_expires_in_minutes,
+            )
+            result = get_check_query_result("zulip", 2)
+            check_multiuse_invite_link_query_result(result)
+            check_zulip_realm_query_result(result)
+            MultiuseInvite.objects.all().delete()
 
-        email = self.nonreg_email("alice")
-        self.submit_realm_creation_form(
-            email, realm_subdomain="custom-test", realm_name="Zulip test"
-        )
-        result = query_result_from_before(email, 1)
-        check_realm_creation_query_result(result, email)
+            do_send_realm_reactivation_email(get_realm("zulip"), acting_user=None)
+            result = get_check_query_result("zulip", 2)
+            check_realm_reactivation_link_query_result(result)
+            check_zulip_realm_query_result(result)
 
-        invite_expires_in_minutes = 10 * 24 * 60
-        do_create_multiuse_invite_link(
-            self.example_user("hamlet"),
-            invited_as=1,
-            invite_expires_in_minutes=invite_expires_in_minutes,
-        )
-        result = query_result_from_before("zulip", 2)
-        check_multiuse_invite_link_query_result(result)
-        check_zulip_realm_query_result(result)
-        MultiuseInvite.objects.all().delete()
+            lear_nonreg_email = "newguy@lear.org"
+            self.client_post("/accounts/home/", {"email": lear_nonreg_email}, subdomain="lear")
+            result = get_check_query_result(lear_nonreg_email, 1)
+            check_preregistration_user_query_result(result, lear_nonreg_email)
+            check_lear_realm_query_result(result)
 
-        do_send_realm_reactivation_email(get_realm("zulip"), acting_user=None)
-        result = query_result_from_before("zulip", 2)
-        check_realm_reactivation_link_query_result(result)
-        check_zulip_realm_query_result(result)
-
-        lear_nonreg_email = "newguy@lear.org"
-        self.client_post("/accounts/home/", {"email": lear_nonreg_email}, subdomain="lear")
-        result = query_result_from_before(lear_nonreg_email, 1)
-        check_preregistration_user_query_result(result, lear_nonreg_email)
-        check_lear_realm_query_result(result)
-
-        self.login_user(lear_user)
-        create_invitation("general", "newguy2@lear.org", lear_realm)
-        result = query_result_from_before("newguy2@lear.org", 1, lear_realm.string_id)
-        check_preregistration_user_query_result(result, "newguy2@lear.org", invite=True)
-        check_lear_realm_query_result(result)
+            self.login_user(lear_user)
+            create_invitation("general", "newguy2@lear.org", lear_realm)
+            result = get_check_query_result("newguy2@lear.org", 1, lear_realm.string_id)
+            check_preregistration_user_query_result(result, "newguy2@lear.org", invite=True)
+            check_lear_realm_query_result(result)
 
     def test_get_org_type_display_name(self) -> None:
         self.assertEqual(get_org_type_display_name(Realm.ORG_TYPES["business"]["id"]), "Business")
@@ -502,7 +345,7 @@ class TestSupportEndpoint(ZulipTestCase):
         """
         Unspecified org type is special in that it is marked to not be shown
         on the registration page (because organitions are not meant to be able to choose it),
-        but should be correctly shown at the /support/ endpoint.
+        but should be correctly shown at the /support endpoint.
         """
         realm = get_realm("zulip")
 
@@ -520,50 +363,38 @@ class TestSupportEndpoint(ZulipTestCase):
             result,
         )
 
-    def test_change_billing_modality(self) -> None:
-        realm = get_realm("zulip")
+    @mock.patch("analytics.views.support.update_billing_method_of_current_plan")
+    def test_change_billing_method(self, m: mock.Mock) -> None:
         cordelia = self.example_user("cordelia")
         self.login_user(cordelia)
+
         result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{realm.id}", "billing_method": "charge_automatically"},
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
         )
         self.assertEqual(result.status_code, 302)
         self.assertEqual(result["Location"], "/login/")
 
-        customer = Customer.objects.create(realm=realm, stripe_customer_id="cus_12345")
-        CustomerPlan.objects.create(
-            customer=customer,
-            status=CustomerPlan.ACTIVE,
-            billing_cycle_anchor=timezone_now(),
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            tier=CustomerPlan.TIER_CLOUD_STANDARD,
-        )
-
         iago = self.example_user("iago")
         self.login_user(iago)
 
         result = self.client_post(
             "/activity/support",
-            {"realm_id": f"{realm.id}", "billing_modality": "charge_automatically"},
+            {"realm_id": f"{iago.realm_id}", "billing_method": "charge_automatically"},
         )
+        m.assert_called_once_with(get_realm("zulip"), charge_automatically=True, acting_user=iago)
         self.assert_in_success_response(
-            ["Billing collection method of zulip updated to charge automatically"], result
+            ["Billing method of zulip updated to charge automatically"], result
         )
-        plan = get_current_plan_by_realm(realm)
-        assert plan is not None
-        self.assertEqual(plan.charge_automatically, True)
+
+        m.reset_mock()
 
         result = self.client_post(
-            "/activity/support", {"realm_id": f"{realm.id}", "billing_modality": "send_invoice"}
+            "/activity/support", {"realm_id": f"{iago.realm_id}", "billing_method": "send_invoice"}
         )
+        m.assert_called_once_with(get_realm("zulip"), charge_automatically=False, acting_user=iago)
         self.assert_in_success_response(
-            ["Billing collection method of zulip updated to send invoice"], result
+            ["Billing method of zulip updated to pay by invoice"], result
         )
-        realm.refresh_from_db()
-        plan = get_current_plan_by_realm(realm)
-        assert plan is not None
-        self.assertEqual(plan.charge_automatically, False)
 
     def test_change_realm_plan_type(self) -> None:
         cordelia = self.example_user("cordelia")
@@ -584,7 +415,7 @@ class TestSupportEndpoint(ZulipTestCase):
             )
             m.assert_called_once_with(get_realm("zulip"), 2, acting_user=iago)
             self.assert_in_success_response(
-                ["Plan type of zulip changed from Self-hosted to Limited"], result
+                ["Plan type of zulip changed from self-hosted to limited"], result
             )
 
         with mock.patch("analytics.views.support.do_change_realm_plan_type") as m:
@@ -593,7 +424,7 @@ class TestSupportEndpoint(ZulipTestCase):
             )
             m.assert_called_once_with(get_realm("zulip"), 10, acting_user=iago)
             self.assert_in_success_response(
-                ["Plan type of zulip changed from Self-hosted to Plus"], result
+                ["Plan type of zulip changed from self-hosted to plus"], result
             )
 
     def test_change_org_type(self) -> None:
@@ -630,15 +461,14 @@ class TestSupportEndpoint(ZulipTestCase):
         self.assertEqual(result["Location"], "/login/")
 
         iago = self.example_user("iago")
-        self.login_user(iago)
+        self.login("iago")
 
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
-        )
-        self.assert_in_success_response(["Discount for lear changed to 25% from 0%"], result)
-        customer = get_customer_by_realm(lear_realm)
-        assert customer is not None
-        self.assertEqual(customer.default_discount, Decimal(25))
+        with mock.patch("analytics.views.support.attach_discount_to_realm") as m:
+            result = self.client_post(
+                "/activity/support", {"realm_id": f"{lear_realm.id}", "discount": "25"}
+            )
+            m.assert_called_once_with(get_realm("lear"), 25, acting_user=iago)
+            self.assert_in_success_response(["Discount of lear changed to 25% from 0%"], result)
 
     def test_change_sponsorship_status(self) -> None:
         lear_realm = get_realm("lear")
@@ -673,12 +503,8 @@ class TestSupportEndpoint(ZulipTestCase):
         self.assertFalse(customer.sponsorship_pending)
 
     def test_approve_sponsorship(self) -> None:
-        support_admin = self.example_user("iago")
         lear_realm = get_realm("lear")
-        billing_session = RealmBillingSession(
-            user=support_admin, realm=lear_realm, support_session=True
-        )
-        billing_session.update_customer_sponsorship_status(True)
+        update_sponsorship_status(lear_realm, True, acting_user=None)
         king_user = self.lear_user("king")
         king_user.role = UserProfile.ROLE_REALM_OWNER
         king_user.save()
@@ -768,109 +594,79 @@ class TestSupportEndpoint(ZulipTestCase):
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "new-name"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
         result = self.client_post(
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "zulip"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
         result = self.client_post(
             "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "lear"}
         )
         self.assert_in_success_response(
-            ["Subdomain already in use. Please choose a different one."], result
+            ["Subdomain unavailable. Please choose a different one."], result
         )
 
-        # Test renaming to a "reserved" subdomain
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{lear_realm.id}", "new_subdomain": "your-org"}
-        )
-        self.assert_in_success_response(
-            ["Subdomain reserved. Please choose a different one."], result
-        )
-
-    def test_modify_plan_for_downgrade_at_end_of_billing_cycle(self) -> None:
-        realm = get_realm("zulip")
+    def test_downgrade_realm(self) -> None:
         cordelia = self.example_user("cordelia")
         self.login_user(cordelia)
         result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{realm.id}", "modify_plan": "downgrade_at_billing_cycle_end"},
+            "/activity/support", {"realm_id": f"{cordelia.realm_id}", "plan_type": "2"}
         )
         self.assertEqual(result.status_code, 302)
         self.assertEqual(result["Location"], "/login/")
 
-        customer = Customer.objects.create(realm=realm, stripe_customer_id="cus_12345")
-        CustomerPlan.objects.create(
-            customer=customer,
-            status=CustomerPlan.ACTIVE,
-            billing_cycle_anchor=timezone_now(),
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            tier=CustomerPlan.TIER_CLOUD_STANDARD,
-        )
-
         iago = self.example_user("iago")
         self.login_user(iago)
 
-        with self.assertLogs("corporate.stripe", "INFO") as m:
+        with mock.patch("analytics.views.support.downgrade_at_the_end_of_billing_cycle") as m:
             result = self.client_post(
                 "/activity/support",
                 {
-                    "realm_id": f"{realm.id}",
-                    "modify_plan": "downgrade_at_billing_cycle_end",
+                    "realm_id": f"{iago.realm_id}",
+                    "downgrade_method": "downgrade_at_billing_cycle_end",
                 },
             )
+            m.assert_called_once_with(get_realm("zulip"))
             self.assert_in_success_response(
                 ["zulip marked for downgrade at the end of billing cycle"], result
             )
-            plan = get_current_plan_by_realm(realm)
-            assert plan is not None
-            self.assertEqual(plan.status, CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE)
-            expected_log = f"INFO:corporate.stripe:Change plan status: Customer.id: {customer.id}, CustomerPlan.id: {plan.id}, status: {CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE}"
-            self.assertEqual(m.output[0], expected_log)
 
-    def test_modify_plan_for_downgrade_now_without_additional_licenses(self) -> None:
-        realm = get_realm("zulip")
-        cordelia = self.example_user("cordelia")
-        self.login_user(cordelia)
-        result = self.client_post(
-            "/activity/support",
-            {"realm_id": f"{realm.id}", "modify_plan": "downgrade_now_without_additional_licenses"},
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
+        with mock.patch(
+            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
+        ) as m:
+            result = self.client_post(
+                "/activity/support",
+                {
+                    "realm_id": f"{iago.realm_id}",
+                    "downgrade_method": "downgrade_now_without_additional_licenses",
+                },
+            )
+            m.assert_called_once_with(get_realm("zulip"))
+            self.assert_in_success_response(
+                ["zulip downgraded without creating additional invoices"], result
+            )
 
-        customer = Customer.objects.create(realm=realm, stripe_customer_id="cus_12345")
-        plan = CustomerPlan.objects.create(
-            customer=customer,
-            status=CustomerPlan.ACTIVE,
-            billing_cycle_anchor=timezone_now(),
-            billing_schedule=CustomerPlan.BILLING_SCHEDULE_ANNUAL,
-            tier=CustomerPlan.TIER_CLOUD_STANDARD,
-        )
-
-        iago = self.example_user("iago")
-        self.login_user(iago)
-
-        result = self.client_post(
-            "/activity/support",
-            {
-                "realm_id": f"{iago.realm_id}",
-                "modify_plan": "downgrade_now_without_additional_licenses",
-            },
-        )
-        self.assert_in_success_response(
-            ["zulip downgraded without creating additional invoices"], result
-        )
-
-        plan.refresh_from_db()
-        self.assertEqual(plan.status, CustomerPlan.ENDED)
-        realm.refresh_from_db()
-        self.assertEqual(realm.plan_type, Realm.PLAN_TYPE_LIMITED)
+        with mock.patch(
+            "analytics.views.support.downgrade_now_without_creating_additional_invoices"
+        ) as m1:
+            with mock.patch("analytics.views.support.void_all_open_invoices", return_value=1) as m2:
+                result = self.client_post(
+                    "/activity/support",
+                    {
+                        "realm_id": f"{iago.realm_id}",
+                        "downgrade_method": "downgrade_now_void_open_invoices",
+                    },
+                )
+                m1.assert_called_once_with(get_realm("zulip"))
+                m2.assert_called_once_with(get_realm("zulip"))
+                self.assert_in_success_response(
+                    ["zulip downgraded and voided 1 open invoices"], result
+                )
 
     def test_scrub_realm(self) -> None:
         cordelia = self.example_user("cordelia")
@@ -896,26 +692,3 @@ class TestSupportEndpoint(ZulipTestCase):
             result = self.client_post("/activity/support", {"realm_id": f"{lear_realm.id}"})
             self.assert_json_error(result, "Invalid parameters")
             m.assert_not_called()
-
-    def test_delete_user(self) -> None:
-        cordelia = self.example_user("cordelia")
-        hamlet = self.example_user("hamlet")
-        hamlet_email = hamlet.delivery_email
-        realm = get_realm("zulip")
-        self.login_user(cordelia)
-
-        result = self.client_post(
-            "/activity/support", {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id}
-        )
-        self.assertEqual(result.status_code, 302)
-        self.assertEqual(result["Location"], "/login/")
-
-        self.login("iago")
-
-        with mock.patch("analytics.views.support.do_delete_user_preserving_messages") as m:
-            result = self.client_post(
-                "/activity/support",
-                {"realm_id": f"{realm.id}", "delete_user_by_id": hamlet.id},
-            )
-            m.assert_called_once_with(hamlet)
-            self.assert_in_success_response([f"{hamlet_email} in zulip deleted"], result)

analytics/urls.py

@@ -1,23 +1,22 @@
 from typing import List, Union
 
-from django.conf import settings
 from django.conf.urls import include
 from django.urls import path
 from django.urls.resolvers import URLPattern, URLResolver
 
-from analytics.views.installation_activity import (
-    get_installation_activity,
-    get_integrations_activity,
-)
+from analytics.views.installation_activity import get_installation_activity
 from analytics.views.realm_activity import get_realm_activity
 from analytics.views.stats import (
     get_chart_data,
     get_chart_data_for_installation,
     get_chart_data_for_realm,
-    get_chart_data_for_stream,
+    get_chart_data_for_remote_installation,
+    get_chart_data_for_remote_realm,
     stats,
     stats_for_installation,
     stats_for_realm,
+    stats_for_remote_installation,
+    stats_for_remote_realm,
 )
 from analytics.views.support import support
 from analytics.views.user_activity import get_user_activity
@@ -26,31 +25,19 @@ from zerver.lib.rest import rest_path
 i18n_urlpatterns: List[Union[URLPattern, URLResolver]] = [
     # Server admin (user_profile.is_staff) visible stats pages
     path("activity", get_installation_activity),
-    path("activity/integrations", get_integrations_activity),
     path("activity/support", support, name="support"),
     path("realm_activity/<realm_str>/", get_realm_activity),
     path("user_activity/<user_profile_id>/", get_user_activity),
     path("stats/realm/<realm_str>/", stats_for_realm),
     path("stats/installation", stats_for_installation),
+    path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
+    path(
+        "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/", stats_for_remote_realm
+    ),
     # User-visible stats page
     path("stats", stats, name="stats"),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.remote_activity import get_remote_server_activity
-    from analytics.views.stats import stats_for_remote_installation, stats_for_remote_realm
-    from analytics.views.support import remote_servers_support
-
-    i18n_urlpatterns += [
-        path("activity/remote", get_remote_server_activity),
-        path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
-        path(
-            "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/",
-            stats_for_remote_realm,
-        ),
-        path("activity/remote/support", remote_servers_support, name="remote_servers_support"),
-    ]
-
 # These endpoints are a part of the API (V1), which uses:
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
@@ -62,28 +49,18 @@ if settings.ZILENCER_ENABLED:
 v1_api_and_json_patterns = [
     # get data for the graphs at /stats
     rest_path("analytics/chart_data", GET=get_chart_data),
-    rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
     rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
     rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/installation",
+        GET=get_chart_data_for_remote_installation,
+    ),
+    rest_path(
+        "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
+        GET=get_chart_data_for_remote_realm,
+    ),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import (
-        get_chart_data_for_remote_installation,
-        get_chart_data_for_remote_realm,
-    )
-
-    v1_api_and_json_patterns += [
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/installation",
-            GET=get_chart_data_for_remote_installation,
-        ),
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
-            GET=get_chart_data_for_remote_realm,
-        ),
-    ]
-
 i18n_urlpatterns += [
     path("api/v1/", include(v1_api_and_json_patterns)),
     path("json/", include(v1_api_and_json_patterns)),

analytics/views/activity_common.py

@@ -1,20 +1,16 @@
 import re
 import sys
 from datetime import datetime
-from typing import Any, Callable, Collection, Dict, List, Optional, Sequence, Union
-from urllib.parse import urlencode
+from html import escape
+from typing import Any, Collection, Dict, List, Optional, Sequence
 
 from django.conf import settings
-from django.db import connection
 from django.db.backends.utils import CursorWrapper
 from django.template import loader
 from django.urls import reverse
-from markupsafe import Markup
-from psycopg2.sql import Composable
+from markupsafe import Markup as mark_safe
 
-from zerver.lib.pysa import mark_sanitized
-from zerver.lib.url_encoding import append_url_query_string
-from zerver.models import Realm, UserActivity
+from zerver.models import UserActivity
 
 if sys.version_info < (3, 9):  # nocoverage
     from backports import zoneinfo
@@ -31,6 +27,7 @@ if settings.BILLING_ENABLED:
 def make_table(
     title: str, cols: Sequence[str], rows: Sequence[Any], has_row_class: bool = False
 ) -> str:
+
     if not has_row_class:
 
         def fix_row(row: Any) -> Dict[str, Any]:
@@ -48,26 +45,8 @@ def make_table(
     return content
 
 
-def fix_rows(
-    rows: List[List[Any]],
-    i: int,
-    fixup_func: Union[Callable[[str], Markup], Callable[[datetime], str]],
-) -> None:
-    for row in rows:
-        row[i] = fixup_func(row[i])
-
-
-def get_query_data(query: Composable) -> List[List[Any]]:
-    cursor = connection.cursor()
-    cursor.execute(query)
-    rows = cursor.fetchall()
-    rows = list(map(list, rows))
-    cursor.close()
-    return rows
-
-
 def dictfetchall(cursor: CursorWrapper) -> List[Dict[str, Any]]:
-    """Returns all rows from a cursor as a dict"""
+    "Returns all rows from a cursor as a dict"
     desc = cursor.description
     return [dict(zip((col[0] for col in desc), row)) for row in cursor.fetchall()]
 
@@ -79,52 +58,36 @@ def format_date_for_activity_reports(date: Optional[datetime]) -> str:
         return ""
 
 
-def user_activity_link(email: str, user_profile_id: int) -> Markup:
+def user_activity_link(email: str, user_profile_id: int) -> mark_safe:
     from analytics.views.user_activity import get_user_activity
 
     url = reverse(get_user_activity, kwargs=dict(user_profile_id=user_profile_id))
-    return Markup('<a href="{url}">{email}</a>').format(url=url, email=email)
+    email_link = f'<a href="{escape(url)}">{escape(email)}</a>'
+    return mark_safe(email_link)
 
 
-def realm_activity_link(realm_str: str) -> Markup:
+def realm_activity_link(realm_str: str) -> mark_safe:
     from analytics.views.realm_activity import get_realm_activity
 
     url = reverse(get_realm_activity, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}">{realm_str}</a>').format(url=url, realm_str=realm_str)
+    realm_link = f'<a href="{escape(url)}">{escape(realm_str)}</a>'
+    return mark_safe(realm_link)
 
 
-def realm_stats_link(realm_str: str) -> Markup:
+def realm_stats_link(realm_str: str) -> mark_safe:
     from analytics.views.stats import stats_for_realm
 
     url = reverse(stats_for_realm, kwargs=dict(realm_str=realm_str))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
+    stats_link = f'<a href="{escape(url)}"><i class="fa fa-pie-chart"></i>{escape(realm_str)}</a>'
+    return mark_safe(stats_link)
 
 
-def realm_support_link(realm_str: str) -> Markup:
-    support_url = reverse("support")
-    query = urlencode({"q": realm_str})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}"><i class="fa fa-gear"></i></a>').format(url=url)
-
-
-def realm_url_link(realm_str: str) -> Markup:
-    host = Realm.host_for_subdomain(realm_str)
-    url = settings.EXTERNAL_URI_SCHEME + mark_sanitized(host)
-    return Markup('<a href="{url}"><i class="fa fa-home"></i></a>').format(url=url)
-
-
-def remote_installation_stats_link(server_id: int) -> Markup:
+def remote_installation_stats_link(server_id: int, hostname: str) -> mark_safe:
     from analytics.views.stats import stats_for_remote_installation
 
     url = reverse(stats_for_remote_installation, kwargs=dict(remote_server_id=server_id))
-    return Markup('<a href="{url}"><i class="fa fa-pie-chart"></i></a>').format(url=url)
-
-
-def remote_installation_support_link(hostname: str) -> Markup:
-    support_url = reverse("remote_servers_support")
-    query = urlencode({"q": hostname})
-    url = append_url_query_string(support_url, query)
-    return Markup('<a href="{url}"><i class="fa fa-gear"></i></a>').format(url=url)
+    stats_link = f'<a href="{escape(url)}"><i class="fa fa-pie-chart"></i>{escape(hostname)}</a>'
+    return mark_safe(stats_link)
 
 
 def get_user_activity_summary(records: Collection[UserActivity]) -> Dict[str, Any]:

analytics/views/installation_activity.py

@@ -1,5 +1,8 @@
+import itertools
+import time
 from collections import defaultdict
-from typing import Dict, Optional
+from datetime import datetime, timedelta
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 from django.conf import settings
 from django.db import connection
@@ -7,54 +10,54 @@ from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 from django.template import loader
 from django.utils.timezone import now as timezone_now
-from markupsafe import Markup
-from psycopg2.sql import SQL
+from markupsafe import Markup as mark_safe
+from psycopg2.sql import SQL, Composable, Literal
 
 from analytics.lib.counts import COUNT_STATS
 from analytics.views.activity_common import (
     dictfetchall,
-    fix_rows,
     format_date_for_activity_reports,
-    get_query_data,
     make_table,
     realm_activity_link,
     realm_stats_link,
-    realm_support_link,
-    realm_url_link,
+    remote_installation_stats_link,
 )
-from analytics.views.support import get_plan_type_string
+from analytics.views.support import get_plan_name
 from zerver.decorator import require_server_admin
 from zerver.lib.request import has_request_variables
-from zerver.models import Realm, get_org_type_display_name
+from zerver.lib.timestamp import timestamp_to_datetime
+from zerver.models import Realm, UserActivityInterval, UserProfile, get_org_type_display_name
 
 if settings.BILLING_ENABLED:
-    from corporate.lib.analytics import (
+    from corporate.lib.stripe import (
         estimate_annual_recurring_revenue_by_realm,
-        get_realms_with_default_discount_dict,
+        get_realms_to_default_discount_dict,
     )
 
 
-def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
-    # To align with UTC days, we subtract an hour from end_time to
-    # get the start_time, since the hour that starts at midnight was
-    # on the previous day.
+def get_realm_day_counts() -> Dict[str, Dict[str, str]]:
     query = SQL(
         """
         select
             r.string_id,
-            (now()::date - (end_time - interval '1 hour')::date) age,
-            coalesce(sum(value), 0) cnt
-        from zerver_realm r
-        join analytics_realmcount rc on r.id = rc.realm_id
+            (now()::date - date_sent::date) age,
+            count(*) cnt
+        from zerver_message m
+        join zerver_userprofile up on up.id = m.sender_id
+        join zerver_realm r on r.id = up.realm_id
+        join zerver_client c on c.id = m.sending_client_id
         where
-            property = 'messages_sent:is_bot:hour'
+            (not up.is_bot)
         and
-            subgroup = 'false'
+            date_sent > now()::date - interval '8 day'
         and
-            end_time > now()::date - interval '8 day' - interval '1 hour'
+            c.name not in ('zephyr_mirror', 'ZulipMonitoring')
         group by
             r.string_id,
             age
+        order by
+            r.string_id,
+            age
     """
     )
     cursor = connection.cursor()
@@ -66,31 +69,31 @@ def get_realm_day_counts() -> Dict[str, Dict[str, Markup]]:
     for row in rows:
         counts[row["string_id"]][row["age"]] = row["cnt"]
 
-    def format_count(cnt: int, style: Optional[str] = None) -> Markup:
-        if style is not None:
-            good_bad = style
-        elif cnt == min_cnt:
-            good_bad = "bad"
-        elif cnt == max_cnt:
-            good_bad = "good"
-        else:
-            good_bad = "neutral"
-
-        return Markup('<td class="number {good_bad}">{cnt}</td>').format(good_bad=good_bad, cnt=cnt)
-
     result = {}
     for string_id in counts:
         raw_cnts = [counts[string_id].get(age, 0) for age in range(8)]
         min_cnt = min(raw_cnts[1:])
         max_cnt = max(raw_cnts[1:])
 
-        cnts = format_count(raw_cnts[0], "neutral") + Markup().join(map(format_count, raw_cnts[1:]))
+        def format_count(cnt: int, style: Optional[str] = None) -> str:
+            if style is not None:
+                good_bad = style
+            elif cnt == min_cnt:
+                good_bad = "bad"
+            elif cnt == max_cnt:
+                good_bad = "good"
+            else:
+                good_bad = "neutral"
+
+            return f'<td class="number {good_bad}">{cnt}</td>'
+
+        cnts = format_count(raw_cnts[0], "neutral") + "".join(map(format_count, raw_cnts[1:]))
         result[string_id] = dict(cnts=cnts)
 
     return result
 
 
-def realm_summary_table() -> str:
+def realm_summary_table(realm_minutes: Dict[str, float]) -> str:
     now = timezone_now()
 
     query = SQL(
@@ -184,10 +187,19 @@ def realm_summary_table() -> str:
     rows = dictfetchall(cursor)
     cursor.close()
 
+    # Fetch all the realm administrator users
+    realm_owners: Dict[str, List[str]] = defaultdict(list)
+    for up in UserProfile.objects.select_related("realm").filter(
+        role=UserProfile.ROLE_REALM_OWNER,
+        is_active=True,
+    ):
+        realm_owners[up.realm.string_id].append(up.delivery_email)
+
     for row in rows:
         row["date_created_day"] = row["date_created"].strftime("%Y-%m-%d")
         row["age_days"] = int((now - row["date_created"]).total_seconds() / 86400)
         row["is_new"] = row["age_days"] < 12 * 7
+        row["realm_owner_emails"] = ", ".join(realm_owners[row["string_id"]])
 
     # get messages sent per day
     counts = get_realm_day_counts()
@@ -201,10 +213,10 @@ def realm_summary_table() -> str:
     total_arr = 0
     if settings.BILLING_ENABLED:
         estimated_arrs = estimate_annual_recurring_revenue_by_realm()
-        realms_with_default_discount = get_realms_with_default_discount_dict()
+        realms_to_default_discount = get_realms_to_default_discount_dict()
 
         for row in rows:
-            row["plan_type_string"] = get_plan_type_string(row["plan_type"])
+            row["plan_type_string"] = get_plan_name(row["plan_type"])
 
             string_id = row["string_id"]
 
@@ -212,14 +224,14 @@ def realm_summary_table() -> str:
                 row["arr"] = estimated_arrs[string_id]
 
             if row["plan_type"] in [Realm.PLAN_TYPE_STANDARD, Realm.PLAN_TYPE_PLUS]:
-                row["effective_rate"] = 100 - int(realms_with_default_discount.get(string_id, 0))
+                row["effective_rate"] = 100 - int(realms_to_default_discount.get(string_id, 0))
             elif row["plan_type"] == Realm.PLAN_TYPE_STANDARD_FREE:
                 row["effective_rate"] = 0
             elif (
                 row["plan_type"] == Realm.PLAN_TYPE_LIMITED
-                and string_id in realms_with_default_discount
+                and string_id in realms_to_default_discount
             ):
-                row["effective_rate"] = 100 - int(realms_with_default_discount[string_id])
+                row["effective_rate"] = 100 - int(realms_to_default_discount[string_id])
             else:
                 row["effective_rate"] = ""
 
@@ -228,15 +240,29 @@ def realm_summary_table() -> str:
     for row in rows:
         row["org_type_string"] = get_org_type_display_name(row["org_type"])
 
+    # augment data with realm_minutes
+    total_hours = 0.0
+    for row in rows:
+        string_id = row["string_id"]
+        minutes = realm_minutes.get(string_id, 0.0)
+        hours = minutes / 60.0
+        total_hours += hours
+        row["hours"] = str(int(hours))
+        try:
+            row["hours_per_user"] = "{:.1f}".format(hours / row["dau_count"])
+        except Exception:
+            pass
+
     # formatting
     for row in rows:
-        row["realm_url"] = realm_url_link(row["string_id"])
         row["stats_link"] = realm_stats_link(row["string_id"])
-        row["support_link"] = realm_support_link(row["string_id"])
         row["string_id"] = realm_activity_link(row["string_id"])
 
     # Count active sites
-    num_active_sites = sum(row["dau_count"] >= 5 for row in rows)
+    def meets_goal(row: Dict[str, int]) -> bool:
+        return row["dau_count"] >= 5
+
+    num_active_sites = len(list(filter(meets_goal, rows)))
 
     # create totals
     total_dau_count = 0
@@ -255,13 +281,13 @@ def realm_summary_table() -> str:
         org_type_string="",
         effective_rate="",
         arr=total_arr,
-        realm_url="",
         stats_link="",
-        support_link="",
         date_created_day="",
+        realm_owner_emails="",
         dau_count=total_dau_count,
         user_profile_count=total_user_profile_count,
         bot_count=total_bot_count,
+        hours=int(total_hours),
         wau_count=total_wau_count,
     )
 
@@ -272,28 +298,224 @@ def realm_summary_table() -> str:
         dict(
             rows=rows,
             num_active_sites=num_active_sites,
-            utctime=now.strftime("%Y-%m-%d %H:%M %Z"),
+            utctime=now.strftime("%Y-%m-%d %H:%MZ"),
             billing_enabled=settings.BILLING_ENABLED,
         ),
     )
     return content
 
 
-@require_server_admin
-@has_request_variables
-def get_installation_activity(request: HttpRequest) -> HttpResponse:
-    content: str = realm_summary_table()
-    title = "Installation activity"
+def user_activity_intervals() -> Tuple[mark_safe, Dict[str, float]]:
+    day_end = timestamp_to_datetime(time.time())
+    day_start = day_end - timedelta(hours=24)
 
-    return render(
-        request,
-        "analytics/activity_details_template.html",
-        context=dict(data=content, title=title, is_home=True),
+    output = "Per-user online duration for the last 24 hours:\n"
+    total_duration = timedelta(0)
+
+    all_intervals = (
+        UserActivityInterval.objects.filter(
+            end__gte=day_start,
+            start__lte=day_end,
+        )
+        .select_related(
+            "user_profile",
+            "user_profile__realm",
+        )
+        .only(
+            "start",
+            "end",
+            "user_profile__delivery_email",
+            "user_profile__realm__string_id",
+        )
+        .order_by(
+            "user_profile__realm__string_id",
+            "user_profile__delivery_email",
+        )
     )
 
+    by_string_id = lambda row: row.user_profile.realm.string_id
+    by_email = lambda row: row.user_profile.delivery_email
+
+    realm_minutes = {}
+
+    for string_id, realm_intervals in itertools.groupby(all_intervals, by_string_id):
+        realm_duration = timedelta(0)
+        output += f"<hr>{string_id}\n"
+        for email, intervals in itertools.groupby(realm_intervals, by_email):
+            duration = timedelta(0)
+            for interval in intervals:
+                start = max(day_start, interval.start)
+                end = min(day_end, interval.end)
+                duration += end - start
+
+            total_duration += duration
+            realm_duration += duration
+            output += f"  {email:<37}{duration}\n"
+
+        realm_minutes[string_id] = realm_duration.total_seconds() / 60
+
+    output += f"\nTotal duration:                      {total_duration}\n"
+    output += f"\nTotal duration in minutes:           {total_duration.total_seconds() / 60.}\n"
+    output += f"Total duration amortized to a month: {total_duration.total_seconds() * 30. / 60.}"
+    content = mark_safe("<pre>" + output + "</pre>")
+    return content, realm_minutes
+
+
+def ad_hoc_queries() -> List[Dict[str, str]]:
+    def get_page(
+        query: Composable, cols: Sequence[str], title: str, totals_columns: Sequence[int] = []
+    ) -> Dict[str, str]:
+        cursor = connection.cursor()
+        cursor.execute(query)
+        rows = cursor.fetchall()
+        rows = list(map(list, rows))
+        cursor.close()
+
+        def fix_rows(
+            i: int, fixup_func: Union[Callable[[str], mark_safe], Callable[[datetime], str]]
+        ) -> None:
+            for row in rows:
+                row[i] = fixup_func(row[i])
+
+        total_row = []
+        for i, col in enumerate(cols):
+            if col == "Realm":
+                fix_rows(i, realm_activity_link)
+            elif col in ["Last time", "Last visit"]:
+                fix_rows(i, format_date_for_activity_reports)
+            elif col == "Hostname":
+                for row in rows:
+                    row[i] = remote_installation_stats_link(row[0], row[i])
+            if len(totals_columns) > 0:
+                if i == 0:
+                    total_row.append("Total")
+                elif i in totals_columns:
+                    total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
+                else:
+                    total_row.append("")
+        if len(totals_columns) > 0:
+            rows.insert(0, total_row)
+
+        content = make_table(title, cols, rows)
+
+        return dict(
+            content=content,
+            title=title,
+        )
+
+    pages = []
+
+    ###
+
+    for mobile_type in ["Android", "ZulipiOS"]:
+        title = f"{mobile_type} usage"
+
+        query: Composable = SQL(
+            """
+            select
+                realm.string_id,
+                up.id user_id,
+                client.name,
+                sum(count) as hits,
+                max(last_visit) as last_time
+            from zerver_useractivity ua
+            join zerver_client client on client.id = ua.client_id
+            join zerver_userprofile up on up.id = ua.user_profile_id
+            join zerver_realm realm on realm.id = up.realm_id
+            where
+                client.name like {mobile_type}
+            group by string_id, up.id, client.name
+            having max(last_visit) > now() - interval '2 week'
+            order by string_id, up.id, client.name
+        """
+        ).format(
+            mobile_type=Literal(mobile_type),
+        )
+
+        cols = [
+            "Realm",
+            "User id",
+            "Name",
+            "Hits",
+            "Last time",
+        ]
+
+        pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = "Desktop users"
+
+    query = SQL(
+        """
+        select
+            realm.string_id,
+            client.name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            client.name like 'desktop%%'
+        group by string_id, client.name
+        having max(last_visit) > now() - interval '2 week'
+        order by string_id, client.name
+    """
+    )
+
+    cols = [
+        "Realm",
+        "Client",
+        "Hits",
+        "Last time",
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
+
+    title = "Integrations by realm"
+
+    query = SQL(
+        """
+        select
+            realm.string_id,
+            case
+                when query like '%%external%%' then split_part(query, '/', 5)
+                else client.name
+            end client_name,
+            sum(count) as hits,
+            max(last_visit) as last_time
+        from zerver_useractivity ua
+        join zerver_client client on client.id = ua.client_id
+        join zerver_userprofile up on up.id = ua.user_profile_id
+        join zerver_realm realm on realm.id = up.realm_id
+        where
+            (query in ('send_message_backend', '/api/v1/send_message')
+            and client.name not in ('Android', 'ZulipiOS')
+            and client.name not like 'test: Zulip%%'
+            )
+        or
+            query like '%%external%%'
+        group by string_id, client_name
+        having max(last_visit) > now() - interval '2 week'
+        order by string_id, client_name
+    """
+    )
+
+    cols = [
+        "Realm",
+        "Client",
+        "Hits",
+        "Last time",
+    ]
+
+    pages.append(get_page(query, cols, title))
+
+    ###
 
-@require_server_admin
-def get_integrations_activity(request: HttpRequest) -> HttpResponse:
     title = "Integrations by client"
 
     query = SQL(
@@ -330,20 +552,71 @@ def get_integrations_activity(request: HttpRequest) -> HttpResponse:
         "Last time",
     ]
 
-    rows = get_query_data(query)
-    for i, col in enumerate(cols):
-        if col == "Realm":
-            fix_rows(rows, i, realm_activity_link)
-        elif col == "Last time":
-            fix_rows(rows, i, format_date_for_activity_reports)
+    pages.append(get_page(query, cols, title))
+
+    title = "Remote Zulip servers"
+
+    query = SQL(
+        """
+        with icount as (
+            select
+                server_id,
+                max(value) as max_value,
+                max(end_time) as max_end_time
+            from zilencer_remoteinstallationcount
+            where
+                property='active_users:is_bot:day'
+                and subgroup='false'
+            group by server_id
+            ),
+        remote_push_devices as (
+            select server_id, count(distinct(user_id)) as push_user_count from zilencer_remotepushdevicetoken
+            group by server_id
+        )
+        select
+            rserver.id,
+            rserver.hostname,
+            rserver.contact_email,
+            max_value,
+            push_user_count,
+            max_end_time
+        from zilencer_remotezulipserver rserver
+        left join icount on icount.server_id = rserver.id
+        left join remote_push_devices on remote_push_devices.server_id = rserver.id
+        order by max_value DESC NULLS LAST, push_user_count DESC NULLS LAST
+    """
+    )
+
+    cols = [
+        "ID",
+        "Hostname",
+        "Contact email",
+        "Analytics users",
+        "Mobile users",
+        "Last update time",
+    ]
+
+    pages.append(get_page(query, cols, title, totals_columns=[3, 4]))
+
+    return pages
+
+
+@require_server_admin
+@has_request_variables
+def get_installation_activity(request: HttpRequest) -> HttpResponse:
+    duration_content, realm_minutes = user_activity_intervals()
+    counts_content: str = realm_summary_table(realm_minutes)
+    data = [
+        ("Counts", counts_content),
+        ("Durations", duration_content),
+    ]
+    for page in ad_hoc_queries():
+        data.append((page["title"], page["content"]))
+
+    title = "Activity"
 
-    content = make_table(title, cols, rows)
     return render(
         request,
-        "analytics/activity_details_template.html",
-        context=dict(
-            data=content,
-            title=title,
-            is_home=False,
-        ),
+        "analytics/activity.html",
+        context=dict(data=data, title=title, is_home=True),
     )

analytics/views/realm_activity.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from typing import Any, Dict, List, Optional, Set, Tuple
 
 from django.db import connection
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils.timezone import now as timezone_now
@@ -13,7 +13,6 @@ from analytics.views.activity_common import (
     format_date_for_activity_reports,
     get_user_activity_summary,
     make_table,
-    realm_stats_link,
     user_activity_link,
 )
 from zerver.decorator import require_server_admin
@@ -163,13 +162,12 @@ def sent_messages_report(realm: str) -> str:
         "Bots",
     ]
 
-    # Uses index: zerver_message_realm_date_sent
     query = SQL(
         """
         select
             series.day::date,
-            user_messages.humans,
-            user_messages.bots
+            humans.cnt,
+            bots.cnt
         from (
             select generate_series(
                 (now()::date - interval '2 week'),
@@ -180,27 +178,45 @@ def sent_messages_report(realm: str) -> str:
         left join (
             select
                 date_sent::date date_sent,
-                count(*) filter (where not up.is_bot) as humans,
-                count(*) filter (where up.is_bot) as bots
+                count(*) cnt
             from zerver_message m
             join zerver_userprofile up on up.id = m.sender_id
             join zerver_realm r on r.id = up.realm_id
             where
                 r.string_id = %s
             and
-                date_sent > now() - interval '2 week'
+                (not up.is_bot)
             and
-                m.realm_id = r.id
+                date_sent > now() - interval '2 week'
             group by
                 date_sent::date
             order by
                 date_sent::date
-        ) user_messages on
-            series.day = user_messages.date_sent
+        ) humans on
+            series.day = humans.date_sent
+        left join (
+            select
+                date_sent::date date_sent,
+                count(*) cnt
+            from zerver_message m
+            join zerver_userprofile up on up.id = m.sender_id
+            join zerver_realm r on r.id = up.realm_id
+            where
+                r.string_id = %s
+            and
+                up.is_bot
+            and
+                date_sent > now() - interval '2 week'
+            group by
+                date_sent::date
+            order by
+                date_sent::date
+        ) bots on
+            series.day = bots.date_sent
     """
     )
     cursor = connection.cursor()
-    cursor.execute(query, [realm])
+    cursor.execute(query, [realm, realm])
     rows = cursor.fetchall()
     cursor.close()
 
@@ -236,10 +252,8 @@ def get_realm_activity(request: HttpRequest, realm_str: str) -> HttpResponse:
     data += [(page_title, content)]
 
     title = realm_str
-    realm_stats = realm_stats_link(realm_str)
-
     return render(
         request,
         "analytics/activity.html",
-        context=dict(data=data, realm_stats_link=realm_stats, title=title),
+        context=dict(data=data, realm_link=None, title=title),
     )

analytics/views/remote_activity.py

@@ -1,136 +0,0 @@
-from django.http import HttpRequest, HttpResponse
-from django.shortcuts import render
-from psycopg2.sql import SQL
-
-from analytics.views.activity_common import (
-    fix_rows,
-    format_date_for_activity_reports,
-    get_query_data,
-    make_table,
-    remote_installation_stats_link,
-    remote_installation_support_link,
-)
-from corporate.lib.analytics import get_plan_data_by_remote_server
-from zerver.decorator import require_server_admin
-from zilencer.models import get_remote_server_guest_and_non_guest_count
-
-
-@require_server_admin
-def get_remote_server_activity(request: HttpRequest) -> HttpResponse:
-    title = "Remote servers"
-
-    query = SQL(
-        """
-        with icount_id as (
-            select
-                server_id,
-                max(id) as max_count_id
-            from zilencer_remoteinstallationcount
-            where
-                property='active_users:is_bot:day'
-                and subgroup='false'
-            group by server_id
-        ),
-        icount as (
-            select
-                icount_id.server_id,
-                value as latest_value,
-                end_time as latest_end_time
-            from icount_id
-            join zilencer_remoteinstallationcount
-            on max_count_id = zilencer_remoteinstallationcount.id
-        ),
-        mobile_push_forwarded_count as (
-            select
-                server_id,
-                sum(coalesce(value, 0)) as push_forwarded_count
-            from zilencer_remoteinstallationcount
-            where
-                property = 'mobile_pushes_forwarded::day'
-                and end_time >= current_timestamp(0) - interval '7 days'
-            group by server_id
-        ),
-        remote_push_devices as (
-            select
-                server_id,
-                count(distinct(user_id, user_uuid)) as push_user_count
-            from zilencer_remotepushdevicetoken
-            group by server_id
-        )
-        select
-            rserver.id,
-            rserver.hostname,
-            rserver.contact_email,
-            rserver.last_version,
-            latest_value,
-            push_user_count,
-            latest_end_time,
-            push_forwarded_count
-        from zilencer_remotezulipserver rserver
-        left join icount on icount.server_id = rserver.id
-        left join mobile_push_forwarded_count on mobile_push_forwarded_count.server_id = rserver.id
-        left join remote_push_devices on remote_push_devices.server_id = rserver.id
-        where not deactivated
-        order by latest_value DESC NULLS LAST, push_user_count DESC NULLS LAST
-    """
-    )
-
-    cols = [
-        "ID",
-        "Hostname",
-        "Contact email",
-        "Zulip version",
-        "Analytics users",
-        "Mobile users",
-        "Last update time",
-        "Mobile pushes forwarded",
-        "Plan name",
-        "Plan status",
-        "ARR",
-        "Non guest users",
-        "Guest users",
-        "Links",
-    ]
-
-    rows = get_query_data(query)
-    total_row = []
-    totals_columns = [4, 5]
-    plan_data_by_remote_server = get_plan_data_by_remote_server()
-
-    for row in rows:
-        # Add estimated revenue for server
-        server_plan_data = plan_data_by_remote_server.get(row[0])
-        if server_plan_data is None:
-            row.append("---")
-            row.append("---")
-            row.append("---")
-        else:
-            row.append(server_plan_data.current_plan_name)
-            row.append(server_plan_data.current_status)
-            row.append(server_plan_data.annual_revenue)
-        # Add user counts
-        remote_server_counts = get_remote_server_guest_and_non_guest_count(row[0])
-        row.append(remote_server_counts.non_guest_user_count)
-        row.append(remote_server_counts.guest_user_count)
-        # Add links
-        stats = remote_installation_stats_link(row[0])
-        support = remote_installation_support_link(row[1])
-        links = stats + " " + support
-        row.append(links)
-    for i, col in enumerate(cols):
-        if col == "Last update time":
-            fix_rows(rows, i, format_date_for_activity_reports)
-        if i == 0:
-            total_row.append("Total")
-        elif i in totals_columns:
-            total_row.append(str(sum(row[i] for row in rows if row[i] is not None)))
-        else:
-            total_row.append("")
-    rows.insert(0, total_row)
-
-    content = make_table(title, cols, rows)
-    return render(
-        request,
-        "analytics/activity_details_template.html",
-        context=dict(data=content, title=title, is_home=False),
-    )

analytics/views/stats.py

@@ -4,13 +4,12 @@ from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
 
 from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
 from django.shortcuts import render
 from django.utils import translation
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
-from typing_extensions import TypeAlias
 
 from analytics.lib.counts import COUNT_STATS, CountStat
 from analytics.lib.time_utils import time_range
@@ -33,10 +32,9 @@ from zerver.lib.exceptions import JsonableError
 from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_success
-from zerver.lib.streams import access_stream_by_id
 from zerver.lib.timestamp import convert_to_UTC
 from zerver.lib.validator import to_non_negative_int
-from zerver.models import Client, Realm, Stream, UserProfile, get_realm
+from zerver.models import Client, Realm, UserProfile, get_realm
 
 if settings.ZILENCER_ENABLED:
     from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
@@ -51,36 +49,17 @@ def is_analytics_ready(realm: Realm) -> bool:
 def render_stats(
     request: HttpRequest,
     data_url_suffix: str,
-    realm: Optional[Realm],
-    *,
-    title: Optional[str] = None,
+    target_name: str,
     for_installation: bool = False,
     remote: bool = False,
     analytics_ready: bool = True,
 ) -> HttpResponse:
     assert request.user.is_authenticated
-
-    if realm is not None:
-        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
-        guest_users = UserProfile.objects.filter(
-            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
-        ).count()
-        space_used = realm.currently_used_upload_space_bytes()
-        if title:
-            pass
-        else:
-            title = realm.name or realm.string_id
-    else:
-        assert title
-        guest_users = None
-        space_used = None
-
     page_params = dict(
         data_url_suffix=data_url_suffix,
         for_installation=for_installation,
         remote=remote,
-        upload_space_used=space_used,
-        guest_users=guest_users,
+        upload_space_used=request.user.realm.currently_used_upload_space_bytes(),
     )
 
     request_language = get_and_set_request_language(
@@ -95,9 +74,7 @@ def render_stats(
         request,
         "analytics/stats.html",
         context=dict(
-            target_name=title,
-            page_params=page_params,
-            analytics_ready=analytics_ready,
+            target_name=target_name, page_params=page_params, analytics_ready=analytics_ready
         ),
     )
 
@@ -110,7 +87,9 @@ def stats(request: HttpRequest) -> HttpResponse:
         # TODO: Make @zulip_login_required pass the UserProfile so we
         # can use @require_member_or_admin
         raise JsonableError(_("Not allowed for guest users"))
-    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
+    return render_stats(
+        request, "", realm.name or realm.string_id, analytics_ready=is_analytics_ready(realm)
+    )
 
 
 @require_server_admin
@@ -124,7 +103,7 @@ def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
     return render_stats(
         request,
         f"/realm/{realm_str}",
-        realm,
+        realm.name or realm.string_id,
         analytics_ready=is_analytics_ready(realm),
     )
 
@@ -139,8 +118,7 @@ def stats_for_remote_realm(
     return render_stats(
         request,
         f"/remote/{server.id}/realm/{remote_realm_id}",
-        None,
-        title=f"Realm {remote_realm_id} on server {server.hostname}",
+        f"Realm {remote_realm_id} on server {server.hostname}",
     )
 
 
@@ -157,21 +135,6 @@ def get_chart_data_for_realm(
     return get_chart_data(request, user_profile, realm=realm, **kwargs)
 
 
-@require_non_guest_user
-@has_request_variables
-def get_chart_data_for_stream(
-    request: HttpRequest, /, user_profile: UserProfile, stream_id: int
-) -> HttpResponse:
-    stream, ignored_sub = access_stream_by_id(
-        user_profile,
-        stream_id,
-        require_active=True,
-        allow_realm_admin=True,
-    )
-
-    return get_chart_data(request, user_profile, stream=stream)
-
-
 @require_server_admin_api
 @has_request_variables
 def get_chart_data_for_remote_realm(
@@ -196,8 +159,7 @@ def get_chart_data_for_remote_realm(
 
 @require_server_admin
 def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation", for_installation=True)
+    return render_stats(request, "/installation", "installation", True)
 
 
 @require_server_admin
@@ -207,10 +169,9 @@ def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -
     return render_stats(
         request,
         f"/remote/{server.id}/installation",
-        None,
-        title=f"remote installation {server.hostname}",
-        for_installation=True,
-        remote=True,
+        f"remote installation {server.hostname}",
+        True,
+        True,
     )
 
 
@@ -253,17 +214,13 @@ def get_chart_data(
     min_length: Optional[int] = REQ(converter=to_non_negative_int, default=None),
     start: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
     end: Optional[datetime] = REQ(converter=to_utc_datetime, default=None),
-    # These last several parameters are only used by functions
-    # wrapping get_chart_data; the callers are responsible for
-    # parsing/validation/authorization for them.
     realm: Optional[Realm] = None,
     for_installation: bool = False,
     remote: bool = False,
     remote_realm_id: Optional[int] = None,
     server: Optional["RemoteZulipServer"] = None,
-    stream: Optional[Stream] = None,
 ) -> HttpResponse:
-    TableType: TypeAlias = Union[
+    TableType = Union[
         Type["RemoteInstallationCount"],
         Type[InstallationCount],
         Type["RemoteRealmCount"],
@@ -285,9 +242,7 @@ def get_chart_data(
         else:
             aggregate_table = RealmCount
 
-    tables: Union[
-        Tuple[TableType], Tuple[TableType, Type[UserCount]], Tuple[TableType, Type[StreamCount]]
-    ]
+    tables: Union[Tuple[TableType], Tuple[TableType, Type[UserCount]]]
 
     if chart_name == "number_of_humans":
         stats = [
@@ -316,8 +271,8 @@ def get_chart_data(
             stats[0]: {
                 "public_stream": _("Public streams"),
                 "private_stream": _("Private streams"),
-                "private_message": _("Direct messages"),
-                "huddle_message": _("Group direct messages"),
+                "private_message": _("Private messages"),
+                "huddle_message": _("Group private messages"),
             }
         }
         labels_sort_function = lambda data: sort_by_totals(data["everyone"])
@@ -337,18 +292,8 @@ def get_chart_data(
         subgroup_to_label = {stats[0]: {None: "read"}}
         labels_sort_function = None
         include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_stream":
-        if stream is None:
-            raise JsonableError(
-                _("Missing stream for chart: {chart_name}").format(chart_name=chart_name)
-            )
-        stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
-        tables = (aggregate_table, StreamCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
     else:
-        raise JsonableError(_("Unknown chart name: {chart_name}").format(chart_name=chart_name))
+        raise JsonableError(_("Unknown chart name: {}").format(chart_name))
 
     # Most likely someone using our API endpoint. The /stats page does not
     # pass a start or end in its requests.
@@ -429,7 +374,6 @@ def get_chart_data(
         InstallationCount: "everyone",
         RealmCount: "everyone",
         UserCount: "user",
-        StreamCount: "everyone",
     }
     if settings.ZILENCER_ENABLED:
         aggregation_level[RemoteInstallationCount] = "everyone"
@@ -441,9 +385,6 @@ def get_chart_data(
         RealmCount: realm.id,
         UserCount: user_profile.id,
     }
-    if stream is not None:
-        id_value[StreamCount] = stream.id
-
     if settings.ZILENCER_ENABLED:
         if server is not None:
             id_value[RemoteInstallationCount] = server.id
@@ -474,7 +415,8 @@ def get_chart_data(
 
 
 def sort_by_totals(value_arrays: Dict[str, List[int]]) -> List[str]:
-    totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
+    totals = [(sum(values), label) for label, values in value_arrays.items()]
+    totals.sort(reverse=True)
     return [label for total, label in totals]
 
 
@@ -500,17 +442,17 @@ CountT = TypeVar("CountT", bound=BaseCount)
 
 def table_filtered_to_id(table: Type[CountT], key_id: int) -> QuerySet[CountT]:
     if table == RealmCount:
-        return table._default_manager.filter(realm_id=key_id)
+        return table.objects.filter(realm_id=key_id)
     elif table == UserCount:
-        return table._default_manager.filter(user_id=key_id)
+        return table.objects.filter(user_id=key_id)
     elif table == StreamCount:
-        return table._default_manager.filter(stream_id=key_id)
+        return table.objects.filter(stream_id=key_id)
     elif table == InstallationCount:
-        return table._default_manager.all()
+        return table.objects.all()
     elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
-        return table._default_manager.filter(server_id=key_id)
+        return table.objects.filter(server_id=key_id)
     elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
-        return table._default_manager.filter(realm_id=key_id)
+        return table.objects.filter(realm_id=key_id)
     else:
         raise AssertionError(f"Unknown table: {table}")
 
@@ -542,10 +484,10 @@ def rewrite_client_arrays(value_arrays: Dict[str, List[int]]) -> Dict[str, List[
     for label, array in value_arrays.items():
         mapped_label = client_label_map(label)
         if mapped_label in mapped_arrays:
-            for i in range(len(array)):
+            for i in range(0, len(array)):
                 mapped_arrays[mapped_label][i] += value_arrays[label][i]
         else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
+            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(0, len(array))]
     return mapped_arrays
 
 

analytics/views/support.py

@@ -1,13 +1,13 @@
-from contextlib import suppress
+import urllib
+from dataclasses import dataclass
 from datetime import timedelta
 from decimal import Decimal
-from typing import Any, Dict, Iterable, List, Optional, Union
-from urllib.parse import urlencode, urlsplit
+from typing import Any, Dict, Iterable, List, Optional
+from urllib.parse import urlencode
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import URLValidator
-from django.db.models import Q
 from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import reverse
@@ -25,7 +25,6 @@ from zerver.actions.realm_settings import (
     do_scrub_realm,
     do_send_realm_reactivation_email,
 )
-from zerver.actions.users import do_delete_user_preserving_messages
 from zerver.decorator import require_server_admin
 from zerver.forms import check_subdomain_available
 from zerver.lib.exceptions import JsonableError
@@ -35,53 +34,43 @@ from zerver.lib.subdomains import get_subdomain_from_hostname
 from zerver.lib.validator import check_bool, check_string_in, to_decimal, to_non_negative_int
 from zerver.models import (
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     Realm,
     RealmReactivationStatus,
     UserProfile,
     get_org_type_display_name,
     get_realm,
-    get_user_profile_by_id,
 )
 from zerver.views.invite import get_invitee_emails_set
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.lib.remote_counts import MissingDataError, compute_max_monthly_messages
-    from zilencer.models import RemoteRealm, RemoteZulipServer
-
 if settings.BILLING_ENABLED:
+    from corporate.lib.stripe import approve_sponsorship as do_approve_sponsorship
     from corporate.lib.stripe import (
-        RealmBillingSession,
-        RemoteRealmBillingSession,
-        RemoteServerBillingSession,
-        SupportType,
-        SupportViewRequest,
+        attach_discount_to_realm,
+        downgrade_at_the_end_of_billing_cycle,
+        downgrade_now_without_creating_additional_invoices,
+        get_discount_for_realm,
+        get_latest_seat_count,
+        make_end_of_cycle_updates_if_needed,
+        update_billing_method_of_current_plan,
+        update_sponsorship_status,
+        void_all_open_invoices,
     )
-    from corporate.lib.support import (
-        PlanData,
-        SupportData,
-        get_current_plan_data_for_support_view,
-        get_customer_discount_for_support_view,
-        get_data_for_support_view,
+    from corporate.models import (
+        Customer,
+        CustomerPlan,
+        get_current_plan_by_realm,
+        get_customer_by_realm,
     )
-    from corporate.models import CustomerPlan
 
 
-def get_plan_type_string(plan_type: int) -> str:
+def get_plan_name(plan_type: int) -> str:
     return {
-        Realm.PLAN_TYPE_SELF_HOSTED: "Self-hosted",
-        Realm.PLAN_TYPE_LIMITED: "Limited",
-        Realm.PLAN_TYPE_STANDARD: "Standard",
-        Realm.PLAN_TYPE_STANDARD_FREE: "Standard free",
-        Realm.PLAN_TYPE_PLUS: "Plus",
-        RemoteZulipServer.PLAN_TYPE_SELF_MANAGED: "Self-managed",
-        RemoteZulipServer.PLAN_TYPE_SELF_MANAGED_LEGACY: CustomerPlan.name_from_tier(
-            CustomerPlan.TIER_SELF_HOSTED_LEGACY
-        ),
-        RemoteZulipServer.PLAN_TYPE_COMMUNITY: "Community",
-        RemoteZulipServer.PLAN_TYPE_BUSINESS: "Business",
-        RemoteZulipServer.PLAN_TYPE_ENTERPRISE: "Enterprise",
+        Realm.PLAN_TYPE_SELF_HOSTED: "self-hosted",
+        Realm.PLAN_TYPE_LIMITED: "limited",
+        Realm.PLAN_TYPE_STANDARD: "standard",
+        Realm.PLAN_TYPE_STANDARD_FREE: "open source",
+        Realm.PLAN_TYPE_PLUS: "plus",
     }[plan_type]
 
 
@@ -130,11 +119,10 @@ def get_confirmations(
     return confirmation_dicts
 
 
-VALID_MODIFY_PLAN_METHODS = [
+VALID_DOWNGRADE_METHODS = [
     "downgrade_at_billing_cycle_end",
     "downgrade_now_without_additional_licenses",
     "downgrade_now_void_open_invoices",
-    "upgrade_plan_tier",
 ]
 
 VALID_STATUS_VALUES = [
@@ -142,12 +130,20 @@ VALID_STATUS_VALUES = [
     "deactivated",
 ]
 
-VALID_BILLING_MODALITY_VALUES = [
+VALID_BILLING_METHODS = [
     "send_invoice",
     "charge_automatically",
 ]
 
 
+@dataclass
+class PlanData:
+    customer: Optional["Customer"] = None
+    current_plan: Optional["CustomerPlan"] = None
+    licenses: Optional[int] = None
+    licenses_used: Optional[int] = None
+
+
 @require_server_admin
 @has_request_variables
 def support(
@@ -157,16 +153,15 @@ def support(
     discount: Optional[Decimal] = REQ(default=None, converter=to_decimal),
     new_subdomain: Optional[str] = REQ(default=None),
     status: Optional[str] = REQ(default=None, str_validator=check_string_in(VALID_STATUS_VALUES)),
-    billing_modality: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_BILLING_MODALITY_VALUES)
+    billing_method: Optional[str] = REQ(
+        default=None, str_validator=check_string_in(VALID_BILLING_METHODS)
     ),
     sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
     approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
-    modify_plan: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
+    downgrade_method: Optional[str] = REQ(
+        default=None, str_validator=check_string_in(VALID_DOWNGRADE_METHODS)
     ),
     scrub_realm: bool = REQ(default=False, json_validator=check_bool),
-    delete_user_by_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
     query: Optional[str] = REQ("q", default=None),
     org_type: Optional[int] = REQ(default=None, converter=to_non_negative_int),
 ) -> HttpResponse:
@@ -176,8 +171,6 @@ def support(
         context["success_message"] = request.session["success_message"]
         del request.session["success_message"]
 
-    acting_user = request.user
-    assert isinstance(acting_user, UserProfile)
     if settings.BILLING_ENABLED and request.method == "POST":
         # We check that request.POST only has two keys in it: The
         # realm_id and a field to change.
@@ -190,42 +183,24 @@ def support(
         assert realm_id is not None
         realm = Realm.objects.get(id=realm_id)
 
-        support_view_request = None
-
-        if approve_sponsorship:
-            support_view_request = SupportViewRequest(support_type=SupportType.approve_sponsorship)
-        elif sponsorship_pending is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.update_sponsorship_status,
-                sponsorship_status=sponsorship_pending,
-            )
-        elif discount is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.attach_discount,
-                discount=discount,
-            )
-        elif billing_modality is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.update_billing_modality,
-                billing_modality=billing_modality,
-            )
-        elif modify_plan is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.modify_plan,
-                plan_modification=modify_plan,
-            )
-            if modify_plan == "upgrade_plan_tier":
-                support_view_request["new_plan_tier"] = CustomerPlan.TIER_CLOUD_PLUS
-        elif plan_type is not None:
+        acting_user = request.user
+        assert isinstance(acting_user, UserProfile)
+        if plan_type is not None:
             current_plan_type = realm.plan_type
             do_change_realm_plan_type(realm, plan_type, acting_user=acting_user)
-            msg = f"Plan type of {realm.string_id} changed from {get_plan_type_string(current_plan_type)} to {get_plan_type_string(plan_type)} "
+            msg = f"Plan type of {realm.string_id} changed from {get_plan_name(current_plan_type)} to {get_plan_name(plan_type)} "
             context["success_message"] = msg
         elif org_type is not None:
             current_realm_type = realm.org_type
             do_change_realm_org_type(realm, org_type, acting_user=acting_user)
             msg = f"Org type of {realm.string_id} changed from {get_org_type_display_name(current_realm_type)} to {get_org_type_display_name(org_type)} "
             context["success_message"] = msg
+        elif discount is not None:
+            current_discount = get_discount_for_realm(realm) or 0
+            attach_discount_to_realm(realm, discount, acting_user=acting_user)
+            context[
+                "success_message"
+            ] = f"Discount of {realm.string_id} changed to {discount}% from {current_discount}%."
         elif new_subdomain is not None:
             old_subdomain = realm.string_id
             try:
@@ -249,43 +224,71 @@ def support(
             elif status == "deactivated":
                 do_deactivate_realm(realm, acting_user=acting_user)
                 context["success_message"] = f"{realm.string_id} deactivated."
+        elif billing_method is not None:
+            if billing_method == "send_invoice":
+                update_billing_method_of_current_plan(
+                    realm, charge_automatically=False, acting_user=acting_user
+                )
+                context[
+                    "success_message"
+                ] = f"Billing method of {realm.string_id} updated to pay by invoice."
+            elif billing_method == "charge_automatically":
+                update_billing_method_of_current_plan(
+                    realm, charge_automatically=True, acting_user=acting_user
+                )
+                context[
+                    "success_message"
+                ] = f"Billing method of {realm.string_id} updated to charge automatically."
+        elif sponsorship_pending is not None:
+            if sponsorship_pending:
+                update_sponsorship_status(realm, True, acting_user=acting_user)
+                context["success_message"] = f"{realm.string_id} marked as pending sponsorship."
+            else:
+                update_sponsorship_status(realm, False, acting_user=acting_user)
+                context["success_message"] = f"{realm.string_id} is no longer pending sponsorship."
+        elif approve_sponsorship:
+            do_approve_sponsorship(realm, acting_user=acting_user)
+            context["success_message"] = f"Sponsorship approved for {realm.string_id}"
+        elif downgrade_method is not None:
+            if downgrade_method == "downgrade_at_billing_cycle_end":
+                downgrade_at_the_end_of_billing_cycle(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} marked for downgrade at the end of billing cycle"
+            elif downgrade_method == "downgrade_now_without_additional_licenses":
+                downgrade_now_without_creating_additional_invoices(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} downgraded without creating additional invoices"
+            elif downgrade_method == "downgrade_now_void_open_invoices":
+                downgrade_now_without_creating_additional_invoices(realm)
+                voided_invoices_count = void_all_open_invoices(realm)
+                context[
+                    "success_message"
+                ] = f"{realm.string_id} downgraded and voided {voided_invoices_count} open invoices"
         elif scrub_realm:
             do_scrub_realm(realm, acting_user=acting_user)
             context["success_message"] = f"{realm.string_id} scrubbed."
-        elif delete_user_by_id:
-            user_profile_for_deletion = get_user_profile_by_id(delete_user_by_id)
-            user_email = user_profile_for_deletion.delivery_email
-            assert user_profile_for_deletion.realm == realm
-            do_delete_user_preserving_messages(user_profile_for_deletion)
-            context["success_message"] = f"{user_email} in {realm.subdomain} deleted."
-
-        if support_view_request is not None:
-            billing_session = RealmBillingSession(
-                user=acting_user, realm=realm, support_session=True
-            )
-            success_message = billing_session.process_support_view_request(support_view_request)
-            context["success_message"] = success_message
 
     if query:
         key_words = get_invitee_emails_set(query)
 
-        case_insensitive_users_q = Q()
-        for key_word in key_words:
-            case_insensitive_users_q |= Q(delivery_email__iexact=key_word)
-        users = set(UserProfile.objects.filter(case_insensitive_users_q))
+        users = set(UserProfile.objects.filter(delivery_email__in=key_words))
         realms = set(Realm.objects.filter(string_id__in=key_words))
 
         for key_word in key_words:
             try:
                 URLValidator()(key_word)
-                parse_result = urlsplit(key_word)
+                parse_result = urllib.parse.urlparse(key_word)
                 hostname = parse_result.hostname
                 assert hostname is not None
                 if parse_result.port:
                     hostname = f"{hostname}:{parse_result.port}"
                 subdomain = get_subdomain_from_hostname(hostname)
-                with suppress(Realm.DoesNotExist):
+                try:
                     realms.add(get_realm(subdomain))
+                except Realm.DoesNotExist:
+                    pass
             except ValidationError:
                 users.update(UserProfile.objects.filter(full_name__iexact=key_word))
 
@@ -301,20 +304,11 @@ def support(
             user.id for user in PreregistrationUser.objects.filter(email__in=key_words)
         ]
         confirmations += get_confirmations(
-            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION],
+            [Confirmation.USER_REGISTRATION, Confirmation.INVITATION, Confirmation.REALM_CREATION],
             preregistration_user_ids,
             hostname=request.get_host(),
         )
 
-        preregistration_realm_ids = [
-            user.id for user in PreregistrationRealm.objects.filter(email__in=key_words)
-        ]
-        confirmations += get_confirmations(
-            [Confirmation.REALM_CREATION],
-            preregistration_realm_ids,
-            hostname=request.get_host(),
-        )
-
         multiuse_invite_ids = [
             invite.id for invite in MultiuseInvite.objects.filter(realm__in=realms)
         ]
@@ -342,9 +336,22 @@ def support(
         )
         plan_data: Dict[int, PlanData] = {}
         for realm in all_realms:
-            billing_session = RealmBillingSession(user=None, realm=realm)
-            realm_plan_data = get_current_plan_data_for_support_view(billing_session)
-            plan_data[realm.id] = realm_plan_data
+            current_plan = get_current_plan_by_realm(realm)
+            plan_data[realm.id] = PlanData(
+                customer=get_customer_by_realm(realm),
+                current_plan=current_plan,
+            )
+            if current_plan is not None:
+                new_plan, last_ledger_entry = make_end_of_cycle_updates_if_needed(
+                    current_plan, timezone_now()
+                )
+                if last_ledger_entry is not None:
+                    if new_plan is not None:
+                        plan_data[realm.id].current_plan = new_plan
+                    else:
+                        plan_data[realm.id].current_plan = current_plan
+                    plan_data[realm.id].licenses = last_ledger_entry.licenses
+                    plan_data[realm.id].licenses_used = get_latest_seat_count(realm)
         context["plan_data"] = plan_data
 
     def get_realm_owner_emails_as_string(realm: Realm) -> str:
@@ -363,7 +370,7 @@ def support(
 
     context["get_realm_owner_emails_as_string"] = get_realm_owner_emails_as_string
     context["get_realm_admin_emails_as_string"] = get_realm_admin_emails_as_string
-    context["get_discount"] = get_customer_discount_for_support_view
+    context["get_discount_for_realm"] = get_discount_for_realm
     context["get_org_type_display_name"] = get_org_type_display_name
     context["realm_icon_url"] = realm_icon_url
     context["Confirmation"] = Confirmation
@@ -372,152 +379,3 @@ def support(
     )
 
     return render(request, "analytics/support.html", context=context)
-
-
-def get_remote_servers_for_support(
-    email_to_search: Optional[str], hostname_to_search: Optional[str]
-) -> List["RemoteZulipServer"]:
-    if not email_to_search and not hostname_to_search:
-        return []
-
-    remote_servers_query = RemoteZulipServer.objects.order_by("id").prefetch_related(
-        "remoterealm_set"
-    )
-    if email_to_search:
-        remote_servers_query = remote_servers_query.filter(contact_email__iexact=email_to_search)
-    elif hostname_to_search:
-        remote_servers_query = remote_servers_query.filter(hostname__icontains=hostname_to_search)
-
-    return list(remote_servers_query)
-
-
-@require_server_admin
-@has_request_variables
-def remote_servers_support(
-    request: HttpRequest,
-    query: Optional[str] = REQ("q", default=None),
-    remote_server_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-    remote_realm_id: Optional[int] = REQ(default=None, converter=to_non_negative_int),
-    discount: Optional[Decimal] = REQ(default=None, converter=to_decimal),
-    sponsorship_pending: Optional[bool] = REQ(default=None, json_validator=check_bool),
-    approve_sponsorship: bool = REQ(default=False, json_validator=check_bool),
-    billing_modality: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_BILLING_MODALITY_VALUES)
-    ),
-    modify_plan: Optional[str] = REQ(
-        default=None, str_validator=check_string_in(VALID_MODIFY_PLAN_METHODS)
-    ),
-) -> HttpResponse:
-    context: Dict[str, Any] = {}
-
-    if "success_message" in request.session:
-        context["success_message"] = request.session["success_message"]
-        del request.session["success_message"]
-
-    acting_user = request.user
-    assert isinstance(acting_user, UserProfile)
-    if settings.BILLING_ENABLED and request.method == "POST":
-        # We check that request.POST only has two keys in it:
-        # either the remote_server_id or a remote_realm_id,
-        # and a field to change.
-        keys = set(request.POST.keys())
-        if "csrfmiddlewaretoken" in keys:
-            keys.remove("csrfmiddlewaretoken")
-        if len(keys) != 2:
-            raise JsonableError(_("Invalid parameters"))
-
-        if remote_realm_id is not None:
-            remote_realm_support_request = True
-            remote_realm = RemoteRealm.objects.get(id=remote_realm_id)
-        else:
-            assert remote_server_id is not None
-            remote_realm_support_request = False
-            remote_server = RemoteZulipServer.objects.get(id=remote_server_id)
-
-        support_view_request = None
-
-        if approve_sponsorship:
-            support_view_request = SupportViewRequest(support_type=SupportType.approve_sponsorship)
-        elif sponsorship_pending is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.update_sponsorship_status,
-                sponsorship_status=sponsorship_pending,
-            )
-        elif discount is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.attach_discount,
-                discount=discount,
-            )
-        elif billing_modality is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.update_billing_modality,
-                billing_modality=billing_modality,
-            )
-        elif modify_plan is not None:
-            support_view_request = SupportViewRequest(
-                support_type=SupportType.modify_plan,
-                plan_modification=modify_plan,
-            )
-        if support_view_request is not None:
-            if remote_realm_support_request:
-                success_message = RemoteRealmBillingSession(
-                    support_staff=acting_user, remote_realm=remote_realm
-                ).process_support_view_request(support_view_request)
-            else:
-                success_message = RemoteServerBillingSession(
-                    support_staff=acting_user, remote_server=remote_server
-                ).process_support_view_request(support_view_request)
-            context["success_message"] = success_message
-
-    email_to_search = None
-    hostname_to_search = None
-    if query:
-        if "@" in query:
-            email_to_search = query
-        else:
-            hostname_to_search = query
-
-    remote_servers = get_remote_servers_for_support(
-        email_to_search=email_to_search, hostname_to_search=hostname_to_search
-    )
-    remote_server_to_max_monthly_messages: Dict[int, Union[int, str]] = dict()
-    server_support_data: Dict[int, SupportData] = {}
-    realm_support_data: Dict[int, SupportData] = {}
-    remote_realms: Dict[int, List[RemoteRealm]] = {}
-    for remote_server in remote_servers:
-        # Get remote realms attached to remote server
-        remote_realms_for_server = list(
-            remote_server.remoterealm_set.exclude(is_system_bot_realm=True)
-        )
-        remote_realms[remote_server.id] = remote_realms_for_server
-        # Get plan data for remote realms
-        for remote_realm in remote_realms[remote_server.id]:
-            realm_billing_session = RemoteRealmBillingSession(remote_realm=remote_realm)
-            remote_realm_data = get_data_for_support_view(realm_billing_session)
-            realm_support_data[remote_realm.id] = remote_realm_data
-        # Get plan data for remote server
-        server_billing_session = RemoteServerBillingSession(remote_server=remote_server)
-        remote_server_data = get_data_for_support_view(server_billing_session)
-        server_support_data[remote_server.id] = remote_server_data
-        # Get max monthly messages
-        try:
-            remote_server_to_max_monthly_messages[remote_server.id] = compute_max_monthly_messages(
-                remote_server
-            )
-        except MissingDataError:
-            remote_server_to_max_monthly_messages[remote_server.id] = "Recent data missing"
-
-    context["remote_servers"] = remote_servers
-    context["remote_servers_support_data"] = server_support_data
-    context["remote_server_to_max_monthly_messages"] = remote_server_to_max_monthly_messages
-    context["remote_realms"] = remote_realms
-    context["remote_realms_support_data"] = realm_support_data
-    context["get_plan_type_name"] = get_plan_type_string
-    context["get_org_type_display_name"] = get_org_type_display_name
-    context["SPONSORED_PLAN_TYPE"] = RemoteZulipServer.PLAN_TYPE_COMMUNITY
-
-    return render(
-        request,
-        "analytics/remote_server_support.html",
-        context=context,
-    )

analytics/views/user_activity.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, List, Tuple
 
 from django.conf import settings
-from django.db.models import QuerySet
+from django.db.models.query import QuerySet
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import render
 
@@ -60,7 +60,7 @@ def raw_user_activity_table(records: QuerySet[UserActivity]) -> str:
 def user_activity_summary_table(user_summary: Dict[str, Dict[str, Any]]) -> str:
     rows = []
     for k, v in user_summary.items():
-        if k in ("name", "user_profile_id"):
+        if k == "name" or k == "user_profile_id":
             continue
         client = k
         count = v["count"]

api_docs/construct-narrow.md

@@ -1,133 +0,0 @@
-# Construct a narrow
-
-A **narrow** is a set of filters for Zulip messages, that can be based
-on many different factors (like sender, stream, topic, search
-keywords, etc.). Narrows are used in various places in the the Zulip
-API (most importantly, in the API for fetching messages).
-
-It is simplest to explain the algorithm for encoding a search as a
-narrow using a single example. Consider the following search query
-(written as it would be entered in the Zulip web app's search box).
-It filters for messages sent to stream `announce`, not sent by
-`iago@zulip.com`, and containing the words `cool` and `sunglasses`:
-
-```
-stream:announce -sender:iago@zulip.com cool sunglasses
-```
-
-This query would be JSON-encoded for use in the Zulip API using JSON
-as a list of simple objects, as follows:
-
-```json
-[
-    {
-        "operator": "stream",
-        "operand": "announce"
-    },
-    {
-        "operator": "sender",
-        "operand": "iago@zulip.com",
-        "negated": true
-    },
-    {
-        "operator": "search",
-        "operand": "cool sunglasses"
-    }
-]
-```
-
-The Zulip help center article on [searching for messages](/help/search-for-messages)
-documents the majority of the search/narrow options supported by the
-Zulip API.
-
-Note that many narrows, including all that lack a `stream` or `streams`
-operator, search the current user's personal message history. See
-[searching shared history](/help/search-for-messages#searching-shared-history)
-for details.
-
-**Changes**: In Zulip 7.0 (feature level 177), support was added
-for three filters related to direct messages: `is:dm`, `dm` and
-`dm-including`. The `dm` operator replaced and deprecated the
-`pm-with` operator. The `is:dm` filter replaced and deprecated
-the `is:private` filter. The `dm-including` operator replaced and
-deprecated the `group-pm-with` operator.
-
-The `dm-including` and `group-pm-with` operators return slightly
-different results. For example, `dm-including:1234` returns all
-direct messages (1-on-1 and group) that include the current user
-and the user with the unique user ID of `1234`. On the other hand,
-`group-pm-with:1234` returned only group direct messages that included
-the current user and the user with the unique user ID of `1234`.
-
-Both `dm` and `is:dm` are aliases of `pm-with` and `is:private`
-respectively, and return the same exact results that the deprecated
-filters did.
-
-## Narrows that use IDs
-
-### Message IDs
-
-The `near` and `id` operators, documented in the help center, use message
-IDs for their operands.
-
-* `near:12345`: Search messages around the message with ID `12345`.
-* `id:12345`: Search for only message with ID `12345`.
-
-The message ID operand for the `id` operator may be encoded as either a
-number or a string. The message ID operand for the `near` operator must
-be encoded as a string.
-
-**Changes**: Prior to Zulip 8.0 (feature level 194), the message ID
-operand for the `id` operator needed to be encoded as a string.
-
-
-```json
-[
-    {
-        "operator": "id",
-        "operand": 12345
-    }
-]
-```
-
-### Stream and user IDs
-
-There are a few additional narrow/search options (new in Zulip 2.1)
-that use either stream IDs or user IDs that are not documented in the
-help center because they are primarily useful to API clients:
-
-* `stream:1234`: Search messages sent to the stream with ID `1234`.
-* `sender:1234`: Search messages sent by user ID `1234`.
-* `dm:1234`: Search the direct message conversation between
-  you and user ID `1234`.
-* `dm:1234,5678`: Search the direct message conversation between
-  you, user ID `1234`, and user ID `5678`.
-* `dm-including:1234`: Search all direct messages (1-on-1 and group)
-  that include you and user ID `1234`.
-
-!!! tip ""
-
-    A user ID can be found by [viewing a user's profile][view-profile]
-    in the web or desktop apps. A stream ID can be found when [browsing
-    streams][browse-streams] in the web app via the URL.
-
-The operands for these search options must be encoded either as an
-integer ID or a JSON list of integer IDs. For example, to query
-messages sent by a user 1234 to a direct message thread with yourself,
-user 1234, and user 5678, the correct JSON-encoded query is:
-
-```json
-[
-    {
-        "operator": "dm",
-        "operand": [1234, 5678]
-    },
-    {
-        "operator": "sender",
-        "operand": 1234
-    }
-]
-```
-
-[view-profile]: /help/view-someones-profile
-[browse-streams]: /help/browse-and-subscribe-to-streams

api_docs/create-scheduled-message.md

@@ -1,49 +0,0 @@
-{generate_api_header(/scheduled_messages:post)}
-
-## Usage examples
-
-{start_tabs}
-
-{generate_code_example(python)|/scheduled_messages:post|example}
-
-{generate_code_example(javascript)|/scheduled_messages:post|example}
-
-{tab|curl}
-
-``` curl
-# Create a scheduled stream message
-curl -X POST {{ api_url }}/v1/scheduled_messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=stream \
-    --data-urlencode to=9 \
-    --data-urlencode topic=Hello \
-    --data-urlencode 'content=Nice to meet everyone!' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-# Create a scheduled direct message
-curl -X POST {{ api_url }}/v1/messages \
-    -u BOT_EMAIL_ADDRESS:BOT_API_KEY \
-    --data-urlencode type=direct \
-    --data-urlencode 'to=[9, 10]' \
-    --data-urlencode 'content=Can we meet on Monday?' \
-    --data-urlencode scheduled_delivery_timestamp=3165826990
-
-```
-
-{end_tabs}
-
-## Parameters
-
-{generate_api_arguments_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_parameter_description(/scheduled_messages:post)}
-
-## Response
-
-{generate_return_values_table|zulip.yaml|/scheduled_messages:post}
-
-{generate_response_description(/scheduled_messages:post)}
-
-#### Example response(s)
-
-{generate_code_example|/scheduled_messages:post|fixture}

api_docs/http-headers.md

@@ -1,80 +0,0 @@
-# HTTP headers
-
-This page documents the HTTP headers used by the Zulip API.
-
-Most important is that API clients authenticate to the server using
-HTTP Basic authentication. If you're using the official [Python or
-JavaScript bindings](/api/installation-instructions), this is taken
-care of when you configure said bindings.
-
-Otherwise, see the `curl` example on each endpoint's documentation
-page, which details the request format.
-
-Documented below are additional HTTP headers and header conventions
-generally used by Zulip:
-
-## The `User-Agent` header
-
-Clients are not required to pass a `User-Agent` HTTP header, but we
-highly recommend doing so when writing an integration. It's easy to do
-and it can help save time when debugging issues related to an API
-client.
-
-If provided, the Zulip server will parse the `User-Agent` HTTP header
-in order to identify specific clients and integrations. This
-information is used by the server for logging, [usage
-statistics](/help/analytics), and on rare occasions, for
-backwards-compatibility logic to preserve support for older versions
-of official clients.
-
-Official Zulip clients and integrations use a `User-Agent` that starts
-with something like `ZulipMobile/20.0.103 `, encoding the name of the
-application and it's version.
-
-Zulip's official API bindings have reasonable defaults for
-`User-Agent`. For example, the official Zulip Python bindings have a
-default `User-Agent` starting with `ZulipPython/{version}`, where
-`version` is the version of the library.
-
-You can give your bot/integration its own name by passing the `client`
-parameter when initializing the Python bindings. For example, the
-official Zulip Nagios integration is initialized like this:
-
-``` python
-client = zulip.Client(
-    config_file=opts.config, client=f"ZulipNagios/{VERSION}"
-)
-```
-
-If you are working on an integration that you plan to share outside
-your organization, you can get help picking a good name in
-`#integrations` in the [Zulip development
-community](https://zulip.com/development-community).
-
-## Rate-limiting response headers
-
-To help clients avoid exceeding rate limits, Zulip sets the following
-HTTP headers in all API responses:
-
-* `X-RateLimit-Remaining`: The number of additional requests of this
-  type that the client can send before exceeding its limit.
-* `X-RateLimit-Limit`: The limit that would be applicable to a client
-  that had not made any recent requests of this type. This is useful
-  for designing a client's burst behavior so as to avoid ever reaching
-  a rate limit.
-* `X-RateLimit-Reset`: The time at which the client will no longer
-  have any rate limits applied to it (and thus could do a burst of
-  `X-RateLimit-Limit` requests).
-
-[Zulip's rate limiting rules are configurable][rate-limiting-rules],
-and can vary by server and over time. The default configuration
-currently limits:
-
-* Every user is limited to 200 total API requests per minute.
-* Separate, much lower limits for authentication/login attempts.
-
-When the Zulip server has configured multiple rate limits that apply
-to a given request, the values returned will be for the strictest
-limit.
-
-[rate-limiting-rules]: https://zulip.readthedocs.io/en/latest/production/security-model.html#rate-limiting

api_docs/rest-error-handling.md

@@ -1,34 +0,0 @@
-# Error handling
-
-Zulip's API will always return a JSON format response.
-The HTTP status code indicates whether the request was successful
-(200 = success, 40x = user error, 50x = server error).  Every response
-will contain at least two keys: `msg` (a human-readable error message)
-and `result`, which will be either `error` or `success` (this is
-redundant with the HTTP status code, but is convenient when printing
-responses while debugging).
-
-For some common errors, Zulip provides a `code` attribute.  Where
-present, clients should check `code`, rather than `msg`, when looking
-for specific error conditions, since the `msg` strings are
-internationalized (e.g. the server will send the error message
-translated into French if the user has a French locale).
-
-Each endpoint documents its own unique errors; documented below are
-errors common to many endpoints:
-
-{generate_code_example|/rest-error-handling:post|fixture}
-
-## Ignored Parameters
-
-In JSON success responses, all Zulip REST API endpoints may return
-an array of parameters sent in the request that are not supported
-by that specific endpoint.
-
-While this can be expected, e.g. when sending both current and legacy
-names for a parameter to a Zulip server of unknown version, this often
-indicates either a bug in the client implementation or an attempt to
-configure a new feature while connected to an older Zulip server that
-does not support said feature.
-
-{generate_code_example|/settings:patch|fixture}

api_docs/roles-and-permissions.md

@@ -1,120 +0,0 @@
-# Roles and permissions
-
-Zulip offers several levels of permissions based on a
-[user's role](/help/roles-and-permissions) in a Zulip organization.
-
-Here are some important details to note when working with these
-roles and permissions in Zulip's API:
-
-## A user's role
-
-A user's account data include a `role` property, which contains the
-user's role in the Zulip organization. These roles are encoded as:
-
-* Organization owner: 100
-
-* Organization administrator: 200
-
-* Organization moderator: 300
-
-* Member: 400
-
-* Guest: 600
-
-User account data also include these boolean properties that duplicate
-the related roles above:
-
-* `is_owner` specifying whether the user is an organization owner.
-
-* `is_admin` specifying whether the user is an organization administrator.
-
-* `is_guest` specifying whether the user is a guest user.
-
-These are intended as conveniences for simple clients, and clients
-should prefer using the `role` field, since only that one is updated
-by the [events API](/api/get-events).
-
-Note that [`POST /register`](/api/register-queue) also returns an
-`is_moderator` boolean property specifying whether the current user is
-an organization moderator.
-
-Additionally, user account data include an `is_billing_admin` property
-specifying whether the user is a billing administrator for the Zulip
-organization, which is not related to one of the roles listed above,
-but rather allows for specific permissions related to billing
-administration in [paid Zulip Cloud plans](https://zulip.com/plans/).
-
-### User account data in the API
-
-Endpoints that return the user account data / properties mentioned
-above are:
-
-* [`GET /users`](/api/get-users)
-
-* [`GET /users/{user_id}`](/api/get-user)
-
-* [`GET /users/{email}`](/api/get-user-by-email)
-
-* [`GET /users/me`](/api/get-own-user)
-
-* [`GET /events`](/api/get-events)
-
-* [`POST /register`](/api/register-queue)
-
-Note that the [`POST /register` endpoint](/api/register-queue) returns
-the above boolean properties to describe the role of the current user,
-when `realm_user` is present in `fetch_event_types`.
-
-Additionally, the specific events returned by the
-[`GET /events` endpoint](/api/get-events) containing data related
-to user accounts and roles are the [`realm_user` add
-event](/api/get-events#realm_user-add), and the
-[`realm_user` update event](/api/get-events#realm_user-update).
-
-## Permission levels
-
-Many areas of Zulip are customizable by the roles
-above, such as (but not limited to) [restricting message editing and
-deletion](/help/restrict-message-editing-and-deletion) and
-[streams permissions](/help/stream-permissions). The potential
-permission levels are:
-
-* Everyone / Any user including Guests (least restrictive)
-
-* Members
-
-* Full members
-
-* Moderators
-
-* Administrators
-
-* Owners
-
-* Nobody (most restrictive)
-
-These permission levels and policies in the API are designed to be
-cutoffs in that users with the specified role and above have the
-specified ability or access. For example, a permission level documented
-as 'moderators only' includes organization moderators, administrators,
-and owners.
-
-Note that specific settings and policies in the Zulip API that use these
-permission levels will likely support a subset of those listed above.
-
-## Determining if a user is a full member
-
-When a Zulip organization has set up a [waiting period before new members
-turn into full members](/help/restrict-permissions-of-new-members),
-clients will need to determine if a user's account has aged past the
-organization's waiting period threshold.
-
-The `realm_waiting_period_threshold`, which is the number of days until
-a user's account is treated as a full member, is returned by the
-[`POST /register` endpoint](/api/register-queue) when `realm` is present
-in `fetch_event_types`.
-
-Clients can compare the `realm_waiting_period_threshold` to a user
-accounts's `date_joined` property, which is the time the user account
-was created, to determine if a user has the permissions of a full
-member or a new member.

api_docs/sidebar_index.md

@@ -1,27 +0,0 @@
-## Integrations
-
-* [Overview](/api/integrations-overview)
-* [Incoming webhook integrations](/api/incoming-webhooks-overview)
-* [Hello world walkthrough](/api/incoming-webhooks-walkthrough)
-* [Non-webhook integrations](/api/non-webhook-integrations)
-
-## Interactive bots (beta)
-
-* [Running bots](/api/running-bots)
-* [Deploying bots](/api/deploying-bots)
-* [Writing bots](/api/writing-bots)
-* [Outgoing webhooks](/api/outgoing-webhooks)
-
-## REST API
-
-* [Overview](/api/rest)
-* [Installation instructions](/api/installation-instructions)
-* [API keys](/api/api-keys)
-* [Configuring the Python bindings](/api/configuring-python-bindings)
-* [HTTP headers](/api/http-headers)
-* [Error handling](/api/rest-error-handling)
-* [Roles and permissions](/api/roles-and-permissions)
-* [Client libraries](/api/client-libraries)
-* [API changelog](/api/changelog)
-
-{!rest-endpoints.md!}

babel.config.js

@@ -0,0 +1,25 @@
+"use strict";
+
+module.exports = {
+    plugins: [
+        [
+            "formatjs",
+            {
+                additionalFunctionNames: ["$t", "$t_html"],
+                overrideIdFn: (id, defaultMessage) => defaultMessage,
+            },
+        ],
+    ],
+    presets: [
+        [
+            "@babel/preset-env",
+            {
+                corejs: "3.25",
+                shippedProposals: true,
+                useBuiltIns: "usage",
+            },
+        ],
+        "@babel/typescript",
+    ],
+    sourceType: "unambiguous",
+};

confirmation/migrations/0001_initial.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("contenttypes", "0001_initial"),
     ]

confirmation/migrations/0002_realmcreationkey.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0001_initial"),
     ]

confirmation/migrations/0003_emailchangeconfirmation.py

@@ -3,6 +3,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0002_realmcreationkey"),
     ]

confirmation/migrations/0004_remove_confirmationmanager.py

@@ -3,6 +3,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0003_emailchangeconfirmation"),
     ]

confirmation/migrations/0005_confirmation_realm.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("zerver", "0124_stream_enable_notifications"),
         ("confirmation", "0004_remove_confirmationmanager"),

confirmation/migrations/0006_realmcreationkey_presume_email_valid.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0005_confirmation_realm"),
     ]

confirmation/migrations/0007_add_indexes.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0006_realmcreationkey_presume_email_valid"),
     ]

confirmation/migrations/0008_confirmation_expiry_date.py

@@ -2,6 +2,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0007_add_indexes"),
     ]

confirmation/migrations/0009_confirmation_expiry_date_backfill.py

@@ -5,7 +5,7 @@ from datetime import timedelta
 
 from django.conf import settings
 from django.db import migrations, transaction
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+from django.db.backends.postgresql.schema import BaseDatabaseSchemaEditor
 from django.db.migrations.state import StateApps
 
 

confirmation/migrations/0010_alter_confirmation_expiry_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0009_confirmation_expiry_date_backfill"),
     ]

confirmation/migrations/0011_alter_confirmation_expiry_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("confirmation", "0010_alter_confirmation_expiry_date"),
     ]

confirmation/models.py

@@ -1,10 +1,10 @@
 # Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>
 
 __revision__ = "$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $"
+import datetime
 import secrets
 from base64 import b32encode
-from datetime import timedelta
-from typing import List, Mapping, Optional, Union, cast
+from typing import List, Mapping, Optional, Union
 from urllib.parse import urljoin
 
 from django.conf import settings
@@ -13,31 +13,23 @@ from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.db.models import CASCADE
 from django.http import HttpRequest, HttpResponse
-from django.template.response import TemplateResponse
+from django.shortcuts import render
 from django.urls import reverse
 from django.utils.timezone import now as timezone_now
-from typing_extensions import TypeAlias, override
 
 from confirmation import settings as confirmation_settings
 from zerver.lib.types import UnspecifiedValue
 from zerver.models import (
     EmailChangeStatus,
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     Realm,
     RealmReactivationStatus,
     UserProfile,
 )
 
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import (
-        PreregistrationRemoteRealmBillingUser,
-        PreregistrationRemoteServerBillingUser,
-    )
 
-
-class ConfirmationKeyError(Exception):
+class ConfirmationKeyException(Exception):
     WRONG_LENGTH = 1
     EXPIRED = 2
     DOES_NOT_EXIST = 3
@@ -48,13 +40,13 @@ class ConfirmationKeyError(Exception):
 
 
 def render_confirmation_key_error(
-    request: HttpRequest, exception: ConfirmationKeyError
+    request: HttpRequest, exception: ConfirmationKeyException
 ) -> HttpResponse:
-    if exception.error_type == ConfirmationKeyError.WRONG_LENGTH:
-        return TemplateResponse(request, "confirmation/link_malformed.html", status=404)
-    if exception.error_type == ConfirmationKeyError.EXPIRED:
-        return TemplateResponse(request, "confirmation/link_expired.html", status=404)
-    return TemplateResponse(request, "confirmation/link_does_not_exist.html", status=404)
+    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
+        return render(request, "confirmation/link_malformed.html", status=404)
+    if exception.error_type == ConfirmationKeyException.EXPIRED:
+        return render(request, "confirmation/link_expired.html", status=404)
+    return render(request, "confirmation/link_does_not_exist.html", status=404)
 
 
 def generate_key() -> str:
@@ -62,21 +54,13 @@ def generate_key() -> str:
     return b32encode(secrets.token_bytes(15)).decode().lower()
 
 
-NoZilencerConfirmationObjT: TypeAlias = Union[
+ConfirmationObjT = Union[
     MultiuseInvite,
-    PreregistrationRealm,
     PreregistrationUser,
     EmailChangeStatus,
     UserProfile,
     RealmReactivationStatus,
 ]
-ZilencerConfirmationObjT: TypeAlias = Union[
-    NoZilencerConfirmationObjT,
-    "PreregistrationRemoteServerBillingUser",
-    "PreregistrationRemoteRealmBillingUser",
-]
-
-ConfirmationObjT = Union[NoZilencerConfirmationObjT, ZilencerConfirmationObjT]
 
 
 def get_object_from_key(
@@ -93,16 +77,16 @@ def get_object_from_key(
 
     # Confirmation keys used to be 40 characters
     if len(confirmation_key) not in (24, 40):
-        raise ConfirmationKeyError(ConfirmationKeyError.WRONG_LENGTH)
+        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
     try:
         confirmation = Confirmation.objects.get(
             confirmation_key=confirmation_key, type__in=confirmation_types
         )
     except Confirmation.DoesNotExist:
-        raise ConfirmationKeyError(ConfirmationKeyError.DOES_NOT_EXIST)
+        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)
 
     if confirmation.expiry_date is not None and timezone_now() > confirmation.expiry_date:
-        raise ConfirmationKeyError(ConfirmationKeyError.EXPIRED)
+        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)
 
     obj = confirmation.content_object
     assert obj is not None
@@ -112,10 +96,10 @@ def get_object_from_key(
     if hasattr(obj, "status") and obj.status in [used_value, revoked_value]:
         # Confirmations where the object has the status attribute are one-time use
         # and are marked after being used (or revoked).
-        raise ConfirmationKeyError(ConfirmationKeyError.EXPIRED)
+        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)
 
     if mark_object_used:
-        # MultiuseInvite objects do not use the STATUS_USED status, since they are
+        # MultiuseInvite objects have no status field, since they are
         # intended to be used more than once.
         assert confirmation.type != Confirmation.MULTIUSE_INVITE
         assert hasattr(obj, "status")
@@ -130,22 +114,12 @@ def create_confirmation_link(
     *,
     validity_in_minutes: Union[Optional[int], UnspecifiedValue] = UnspecifiedValue(),
     url_args: Mapping[str, str] = {},
-    no_associated_realm_object: bool = False,
 ) -> str:
     # validity_in_minutes is an override for the default values which are
     # determined by the confirmation_type - its main purpose is for use
     # in tests which may want to have control over the exact expiration time.
     key = generate_key()
-
-    # Some confirmation objects, like those for realm creation or those used
-    # for the self-hosted management flows, are not associated with a realm
-    # hosted by this Zulip server.
-    if no_associated_realm_object:
-        realm = None
-    else:
-        obj = cast(NoZilencerConfirmationObjT, obj)
-        assert not isinstance(obj, PreregistrationRealm)
-        realm = obj.realm
+    realm = obj.realm
 
     current_time = timezone_now()
     expiry_date = None
@@ -154,9 +128,11 @@ def create_confirmation_link(
             expiry_date = None
         else:
             assert validity_in_minutes is not None
-            expiry_date = current_time + timedelta(minutes=validity_in_minutes)
+            expiry_date = current_time + datetime.timedelta(minutes=validity_in_minutes)
     else:
-        expiry_date = current_time + timedelta(days=_properties[confirmation_type].validity_in_days)
+        expiry_date = current_time + datetime.timedelta(
+            days=_properties[confirmation_type].validity_in_days
+        )
 
     Confirmation.objects.create(
         content_object=obj,
@@ -201,17 +177,14 @@ class Confirmation(models.Model):
     MULTIUSE_INVITE = 6
     REALM_CREATION = 7
     REALM_REACTIVATION = 8
-    REMOTE_SERVER_BILLING_LEGACY_LOGIN = 9
-    REMOTE_REALM_BILLING_LEGACY_LOGIN = 10
     type = models.PositiveSmallIntegerField()
 
+    def __str__(self) -> str:
+        return f"<Confirmation: {self.content_object}>"
+
     class Meta:
         unique_together = ("type", "confirmation_key")
 
-    @override
-    def __str__(self) -> str:
-        return f"{self.content_object!r}"
-
 
 class ConfirmationType:
     def __init__(
@@ -239,13 +212,6 @@ _properties = {
     Confirmation.REALM_CREATION: ConfirmationType("get_prereg_key_and_redirect"),
     Confirmation.REALM_REACTIVATION: ConfirmationType("realm_reactivation"),
 }
-if settings.ZILENCER_ENABLED:
-    _properties[Confirmation.REMOTE_SERVER_BILLING_LEGACY_LOGIN] = ConfirmationType(
-        "remote_billing_legacy_server_from_login_confirmation_link"
-    )
-    _properties[Confirmation.REMOTE_REALM_BILLING_LEGACY_LOGIN] = ConfirmationType(
-        "remote_realm_billing_from_login_confirmation_link"
-    )
 
 
 def one_click_unsubscribe_link(user_profile: UserProfile, email_type: str) -> str:
@@ -274,10 +240,10 @@ def validate_key(creation_key: Optional[str]) -> Optional["RealmCreationKey"]:
     try:
         key_record = RealmCreationKey.objects.get(creation_key=creation_key)
     except RealmCreationKey.DoesNotExist:
-        raise RealmCreationKey.InvalidError
+        raise RealmCreationKey.Invalid()
     time_elapsed = timezone_now() - key_record.date_created
     if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600:
-        raise RealmCreationKey.InvalidError
+        raise RealmCreationKey.Invalid()
     return key_record
 
 
@@ -300,5 +266,5 @@ class RealmCreationKey(models.Model):
     # is theirs, and skip sending mail to it to confirm that.
     presume_email_valid = models.BooleanField(default=False)
 
-    class InvalidError(Exception):
+    class Invalid(Exception):
         pass

corporate/lib/analytics.py

@@ -1,94 +0,0 @@
-from dataclasses import dataclass
-from decimal import Decimal
-from typing import Any, Dict
-
-from django.utils.timezone import now as timezone_now
-
-from corporate.lib.stripe import (
-    RealmBillingSession,
-    RemoteRealmBillingSession,
-    RemoteServerBillingSession,
-)
-from corporate.models import Customer, CustomerPlan
-from zerver.lib.utils import assert_is_not_none
-
-
-@dataclass
-class RemoteActivityPlanData:
-    current_status: str
-    current_plan_name: str
-    annual_revenue: int
-
-
-def get_realms_with_default_discount_dict() -> Dict[str, Decimal]:
-    realms_with_default_discount: Dict[str, Any] = {}
-    customers = (
-        Customer.objects.exclude(default_discount=None)
-        .exclude(default_discount=0)
-        .exclude(realm=None)
-    )
-    for customer in customers:
-        assert customer.realm is not None
-        realms_with_default_discount[customer.realm.string_id] = assert_is_not_none(
-            customer.default_discount
-        )
-    return realms_with_default_discount
-
-
-def estimate_annual_recurring_revenue_by_realm() -> Dict[str, int]:  # nocoverage
-    annual_revenue = {}
-    for plan in CustomerPlan.objects.filter(status=CustomerPlan.ACTIVE).select_related(
-        "customer__realm"
-    ):
-        if plan.customer.realm is not None:
-            # TODO: figure out what to do for plans that don't automatically
-            # renew, but which probably will renew
-            renewal_cents = RealmBillingSession(
-                realm=plan.customer.realm
-            ).get_customer_plan_renewal_amount(plan, timezone_now())
-            if plan.billing_schedule == CustomerPlan.BILLING_SCHEDULE_MONTHLY:
-                renewal_cents *= 12
-            # TODO: Decimal stuff
-            annual_revenue[plan.customer.realm.string_id] = int(renewal_cents / 100)
-    return annual_revenue
-
-
-def get_plan_data_by_remote_server() -> Dict[int, RemoteActivityPlanData]:  # nocoverage
-    remote_server_plan_data: Dict[int, RemoteActivityPlanData] = {}
-    for plan in CustomerPlan.objects.filter(
-        status__lt=CustomerPlan.LIVE_STATUS_THRESHOLD, customer__realm__isnull=True
-    ).select_related("customer__remote_server", "customer__remote_realm"):
-        renewal_cents = 0
-        server_id = None
-
-        if plan.customer.remote_server is not None:
-            server_id = plan.customer.remote_server.id
-            renewal_cents = RemoteServerBillingSession(
-                remote_server=plan.customer.remote_server
-            ).get_customer_plan_renewal_amount(plan, timezone_now())
-        elif plan.customer.remote_realm is not None:
-            server_id = plan.customer.remote_realm.server.id
-            renewal_cents = RemoteRealmBillingSession(
-                remote_realm=plan.customer.remote_realm
-            ).get_customer_plan_renewal_amount(plan, timezone_now())
-
-        assert server_id is not None
-
-        if plan.billing_schedule == CustomerPlan.BILLING_SCHEDULE_MONTHLY:
-            renewal_cents *= 12
-
-        current_data = remote_server_plan_data.get(server_id)
-        if current_data is not None:
-            current_revenue = remote_server_plan_data[server_id].annual_revenue
-            remote_server_plan_data[server_id] = RemoteActivityPlanData(
-                current_status="Multiple plans",
-                current_plan_name="See support view",
-                annual_revenue=current_revenue + int(renewal_cents / 100),
-            )
-        else:
-            remote_server_plan_data[server_id] = RemoteActivityPlanData(
-                current_status=plan.get_plan_status_as_text(),
-                current_plan_name=plan.name,
-                annual_revenue=int(renewal_cents / 100),
-            )
-    return remote_server_plan_data

corporate/lib/decorator.py

@@ -1,187 +0,0 @@
-from functools import wraps
-from typing import Callable, Optional
-from urllib.parse import urlencode, urljoin
-
-from django.conf import settings
-from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
-from django.shortcuts import render
-from django.urls import reverse
-from typing_extensions import Concatenate, ParamSpec
-
-from corporate.lib.remote_billing_util import (
-    RemoteBillingIdentityExpiredError,
-    get_remote_realm_and_user_from_session,
-    get_remote_server_and_user_from_session,
-)
-from corporate.lib.stripe import RemoteRealmBillingSession, RemoteServerBillingSession
-from zerver.lib.exceptions import RemoteBillingAuthenticationError
-from zerver.lib.subdomains import get_subdomain
-from zerver.lib.url_encoding import append_url_query_string
-from zilencer.models import RemoteRealm
-
-ParamT = ParamSpec("ParamT")
-
-
-def is_self_hosting_management_subdomain(request: HttpRequest) -> bool:
-    subdomain = get_subdomain(request)
-    return subdomain == settings.SELF_HOSTING_MANAGEMENT_SUBDOMAIN
-
-
-def self_hosting_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, ParamT], HttpResponse]
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest, /, *args: ParamT.args, **kwargs: ParamT.kwargs
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-        return view_func(request, *args, **kwargs)
-
-    return _wrapped_view_func
-
-
-def authenticated_remote_realm_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, RemoteRealmBillingSession, ParamT], HttpResponse]
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest,
-        /,
-        *args: ParamT.args,
-        **kwargs: ParamT.kwargs,
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-
-        realm_uuid = kwargs.get("realm_uuid")
-        if realm_uuid is not None and not isinstance(realm_uuid, str):  # nocoverage
-            raise TypeError("realm_uuid must be a string or None")
-
-        try:
-            remote_realm, remote_billing_user = get_remote_realm_and_user_from_session(
-                request, realm_uuid
-            )
-        except RemoteBillingIdentityExpiredError as e:
-            # The user had an authenticated session with an identity_dict,
-            # but it expired.
-            # We want to redirect back to the start of their login flow
-            # at their {realm.host}/self-hosted-billing/ with a proper
-            # next parameter to take them back to what they're trying
-            # to access after re-authing.
-            # Note: Theoretically we could take the realm_uuid from the request
-            # path or params to figure out the remote_realm.host for the redirect,
-            # but that would mean leaking that .host value to anyone who knows
-            # the uuid. Therefore we limit ourselves to taking the realm_uuid
-            # from the identity_dict - since that proves that the user at least
-            # previously was successfully authenticated as a billing admin of that
-            # realm.
-            realm_uuid = e.realm_uuid
-            server_uuid = e.server_uuid
-            uri_scheme = e.uri_scheme
-            if realm_uuid is None:
-                # This doesn't make sense - if get_remote_realm_and_user_from_session
-                # found an expired identity dict, it should have had a realm_uuid.
-                raise AssertionError
-
-            assert server_uuid is not None, "identity_dict with realm_uuid must have server_uuid"
-            assert uri_scheme is not None, "identity_dict with realm_uuid must have uri_scheme"
-
-            try:
-                remote_realm = RemoteRealm.objects.get(uuid=realm_uuid, server__uuid=server_uuid)
-            except RemoteRealm.DoesNotExist:
-                # This should be impossible - unless the RemoteRealm existed and somehow the row
-                # was deleted.
-                raise AssertionError
-
-            # Using EXTERNAL_URI_SCHEME means we'll do https:// in production, which is
-            # the sane default - while having http:// in development, which will allow
-            # these redirects to work there for testing.
-            url = urljoin(uri_scheme + remote_realm.host, "/self-hosted-billing/")
-
-            page_type = get_next_page_param_from_request_path(request)
-            if page_type is not None:
-                query = urlencode({"next_page": page_type})
-                url = append_url_query_string(url, query)
-
-            return HttpResponseRedirect(url)
-
-        billing_session = RemoteRealmBillingSession(
-            remote_realm, remote_billing_user=remote_billing_user
-        )
-        return view_func(request, billing_session)
-
-    return _wrapped_view_func
-
-
-def get_next_page_param_from_request_path(request: HttpRequest) -> Optional[str]:
-    # Our endpoint URLs in this subsystem end with something like
-    # /sponsorship or /plans etc.
-    # Therefore we can use this nice property to figure out easily what
-    # kind of page the user is trying to access and find the right value
-    # for the `next` query parameter.
-    path = request.path
-    if path.endswith("/"):
-        path = path[:-1]
-
-    page_type = path.split("/")[-1]
-
-    from corporate.views.remote_billing_page import (
-        VALID_NEXT_PAGES as REMOTE_BILLING_VALID_NEXT_PAGES,
-    )
-
-    if page_type in REMOTE_BILLING_VALID_NEXT_PAGES:
-        return page_type
-
-    # Should be impossible to reach here. If this is reached, it must mean
-    # we have a registered endpoint that doesn't have a VALID_NEXT_PAGES entry
-    # or the parsing logic above is failing.
-    raise AssertionError(f"Unknown page type: {page_type}")
-
-
-def authenticated_remote_server_management_endpoint(
-    view_func: Callable[Concatenate[HttpRequest, RemoteServerBillingSession, ParamT], HttpResponse]
-) -> Callable[Concatenate[HttpRequest, ParamT], HttpResponse]:
-    @wraps(view_func)
-    def _wrapped_view_func(
-        request: HttpRequest,
-        /,
-        *args: ParamT.args,
-        **kwargs: ParamT.kwargs,
-    ) -> HttpResponse:
-        if not is_self_hosting_management_subdomain(request):  # nocoverage
-            return render(request, "404.html", status=404)
-
-        server_uuid = kwargs.get("server_uuid")
-        if not isinstance(server_uuid, str):
-            raise TypeError("server_uuid must be a string")  # nocoverage
-
-        try:
-            remote_server, remote_billing_user = get_remote_server_and_user_from_session(
-                request, server_uuid=server_uuid
-            )
-            if remote_billing_user is None:
-                # This should only be possible if the user hasn't finished the confirmation flow
-                # and doesn't have a fully authenticated session yet. They should not be attempting
-                # to access this endpoint yet.
-                raise RemoteBillingAuthenticationError
-        except (RemoteBillingIdentityExpiredError, RemoteBillingAuthenticationError):
-            # In this flow, we can only redirect to our local "legacy server flow login" page.
-            # That means that we can do it universally whether the user has an expired
-            # identity_dict, or just lacks any form of authentication info at all - there
-            # are no security concerns since this is just a local redirect.
-            url = reverse("remote_billing_legacy_server_login")
-            page_type = get_next_page_param_from_request_path(request)
-            if page_type is not None:
-                query = urlencode({"next_page": page_type})
-                url = append_url_query_string(url, query)
-
-            return HttpResponseRedirect(url)
-
-        assert remote_billing_user is not None
-        billing_session = RemoteServerBillingSession(
-            remote_server, remote_billing_user=remote_billing_user
-        )
-        return view_func(request, billing_session)
-
-    return _wrapped_view_func

corporate/lib/registration.py

@@ -72,13 +72,17 @@ def check_spare_licenses_available_for_adding_new_users(
     realm: Realm, extra_non_guests_count: int = 0, extra_guests_count: int = 0
 ) -> None:
     plan = get_current_plan_by_realm(realm)
-    if plan is None or plan.automanage_licenses or plan.customer.exempt_from_license_number_check:
+    if (
+        plan is None
+        or plan.automanage_licenses
+        or plan.customer.exempt_from_from_license_number_check
+    ):
         return
 
     if plan.licenses() < get_seat_count(
         realm, extra_non_guests_count=extra_non_guests_count, extra_guests_count=extra_guests_count
     ):
-        raise LicenseLimitError
+        raise LicenseLimitError()
 
 
 def check_spare_licenses_available_for_registering_new_user(

corporate/lib/remote_billing_util.py

@@ -1,182 +0,0 @@
-import logging
-from typing import Literal, Optional, Tuple, TypedDict, Union, cast
-
-from django.http import HttpRequest
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-
-from zerver.lib.exceptions import JsonableError, RemoteBillingAuthenticationError
-from zerver.lib.timestamp import datetime_to_timestamp
-from zilencer.models import (
-    RemoteRealm,
-    RemoteRealmBillingUser,
-    RemoteServerBillingUser,
-    RemoteZulipServer,
-)
-
-billing_logger = logging.getLogger("corporate.stripe")
-
-# The sessions are relatively short-lived, so that we can avoid issues
-# with users who have their privileges revoked on the remote server
-# maintaining access to the billing page for too long.
-REMOTE_BILLING_SESSION_VALIDITY_SECONDS = 2 * 60 * 60
-
-
-class RemoteBillingUserDict(TypedDict):
-    user_uuid: str
-    user_email: str
-    user_full_name: str
-
-
-class RemoteBillingIdentityDict(TypedDict):
-    user: RemoteBillingUserDict
-    remote_server_uuid: str
-    remote_realm_uuid: str
-
-    remote_billing_user_id: Optional[int]
-    authenticated_at: int
-    uri_scheme: Literal["http://", "https://"]
-
-    next_page: Optional[str]
-
-
-class LegacyServerIdentityDict(TypedDict):
-    # Currently this has only one field. We can extend this
-    # to add more information as appropriate.
-    remote_server_uuid: str
-
-    remote_billing_user_id: Optional[int]
-    authenticated_at: int
-
-
-class RemoteBillingIdentityExpiredError(Exception):
-    def __init__(
-        self,
-        *,
-        realm_uuid: Optional[str] = None,
-        server_uuid: Optional[str] = None,
-        uri_scheme: Optional[Literal["http://", "https://"]] = None,
-    ) -> None:
-        self.realm_uuid = realm_uuid
-        self.server_uuid = server_uuid
-        self.uri_scheme = uri_scheme
-
-
-def get_identity_dict_from_session(
-    request: HttpRequest,
-    *,
-    realm_uuid: Optional[str],
-    server_uuid: Optional[str],
-) -> Optional[Union[RemoteBillingIdentityDict, LegacyServerIdentityDict]]:
-    if not (realm_uuid or server_uuid):
-        return None
-
-    identity_dicts = request.session.get("remote_billing_identities")
-    if identity_dicts is None:
-        return None
-
-    if realm_uuid is not None:
-        result = identity_dicts.get(f"remote_realm:{realm_uuid}")
-    else:
-        assert server_uuid is not None
-        result = identity_dicts.get(f"remote_server:{server_uuid}")
-
-    if result is None:
-        return None
-    if (
-        datetime_to_timestamp(timezone_now()) - result["authenticated_at"]
-        > REMOTE_BILLING_SESSION_VALIDITY_SECONDS
-    ):
-        # In this case we raise, because callers want to catch this as an explicitly
-        # different scenario from the user not being authenticated, to handle it nicely
-        # by redirecting them to their login page.
-        raise RemoteBillingIdentityExpiredError(
-            realm_uuid=result.get("remote_realm_uuid"),
-            server_uuid=result.get("remote_server_uuid"),
-            uri_scheme=result.get("uri_scheme"),
-        )
-
-    return result
-
-
-def get_remote_realm_and_user_from_session(
-    request: HttpRequest,
-    realm_uuid: Optional[str],
-) -> Tuple[RemoteRealm, RemoteRealmBillingUser]:
-    # Cannot use isinstance with TypeDicts, to make mypy know
-    # which of the TypedDicts in the Union this is - so just cast it.
-    identity_dict = cast(
-        Optional[RemoteBillingIdentityDict],
-        get_identity_dict_from_session(request, realm_uuid=realm_uuid, server_uuid=None),
-    )
-
-    if identity_dict is None:
-        raise RemoteBillingAuthenticationError
-
-    remote_server_uuid = identity_dict["remote_server_uuid"]
-    remote_realm_uuid = identity_dict["remote_realm_uuid"]
-
-    try:
-        remote_realm = RemoteRealm.objects.get(
-            uuid=remote_realm_uuid, server__uuid=remote_server_uuid
-        )
-    except RemoteRealm.DoesNotExist:
-        raise AssertionError(
-            "The remote realm is missing despite being in the RemoteBillingIdentityDict"
-        )
-
-    if (
-        remote_realm.registration_deactivated
-        or remote_realm.realm_deactivated
-        or remote_realm.server.deactivated
-    ):
-        raise JsonableError(_("Registration is deactivated"))
-
-    remote_billing_user_id = identity_dict["remote_billing_user_id"]
-    # We only put IdentityDicts with remote_billing_user_id in the session in this flow,
-    # because the RemoteRealmBillingUser already exists when this is inserted into the session
-    # at the end of authentication.
-    assert remote_billing_user_id is not None
-
-    try:
-        remote_billing_user = RemoteRealmBillingUser.objects.get(
-            id=remote_billing_user_id, remote_realm=remote_realm
-        )
-    except RemoteRealmBillingUser.DoesNotExist:
-        raise AssertionError
-
-    return remote_realm, remote_billing_user
-
-
-def get_remote_server_and_user_from_session(
-    request: HttpRequest,
-    server_uuid: str,
-) -> Tuple[RemoteZulipServer, Optional[RemoteServerBillingUser]]:
-    identity_dict: Optional[LegacyServerIdentityDict] = get_identity_dict_from_session(
-        request, realm_uuid=None, server_uuid=server_uuid
-    )
-
-    if identity_dict is None:
-        raise RemoteBillingAuthenticationError
-
-    remote_server_uuid = identity_dict["remote_server_uuid"]
-    try:
-        remote_server = RemoteZulipServer.objects.get(uuid=remote_server_uuid)
-    except RemoteZulipServer.DoesNotExist:
-        raise JsonableError(_("Invalid remote server."))
-
-    if remote_server.deactivated:
-        raise JsonableError(_("Registration is deactivated"))
-
-    remote_billing_user_id = identity_dict.get("remote_billing_user_id")
-    if remote_billing_user_id is None:
-        return remote_server, None
-
-    try:
-        remote_billing_user = RemoteServerBillingUser.objects.get(
-            id=remote_billing_user_id, remote_server=remote_server
-        )
-    except RemoteServerBillingUser.DoesNotExist:
-        remote_billing_user = None
-
-    return remote_server, remote_billing_user

corporate/lib/stripe_event_handler.py

@@ -1,18 +1,17 @@
 import logging
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Union
 
 import stripe
 from django.conf import settings
 
 from corporate.lib.stripe import (
     BillingError,
-    InvalidPlanUpgradeError,
-    RealmBillingSession,
-    RemoteRealmBillingSession,
-    RemoteServerBillingSession,
     UpgradeWithExistingPlanError,
+    ensure_realm_does_not_have_active_plan,
+    process_initial_upgrade,
+    update_or_create_stripe_customer,
 )
-from corporate.models import Customer, CustomerPlan, Event, PaymentIntent, Session
+from corporate.models import Event, PaymentIntent, Session
 from zerver.models import get_active_user_profile_by_id_in_realm
 
 billing_logger = logging.getLogger("corporate.stripe")
@@ -63,20 +62,6 @@ def error_handler(
     return wrapper
 
 
-def get_billing_session_for_stripe_webhook(
-    customer: Customer, user_id: Optional[str]
-) -> Union[RealmBillingSession, RemoteRealmBillingSession, RemoteServerBillingSession]:
-    if customer.remote_realm is not None:  # nocoverage
-        return RemoteRealmBillingSession(customer.remote_realm)
-    elif customer.remote_server is not None:  # nocoverage
-        return RemoteServerBillingSession(customer.remote_server)
-    else:
-        assert user_id is not None
-        assert customer.realm is not None
-        user = get_active_user_profile_by_id_in_realm(int(user_id), customer.realm)
-        return RealmBillingSession(user)
-
-
 @error_handler
 def handle_checkout_session_completed_event(
     stripe_session: stripe.checkout.Session, session: Session
@@ -84,20 +69,47 @@ def handle_checkout_session_completed_event(
     session.status = Session.COMPLETED
     session.save()
 
-    assert isinstance(stripe_session.setup_intent, str)
-    assert stripe_session.metadata is not None
     stripe_setup_intent = stripe.SetupIntent.retrieve(stripe_session.setup_intent)
-    billing_session = get_billing_session_for_stripe_webhook(
-        session.customer, stripe_session.metadata.get("user_id")
-    )
+    assert session.customer.realm is not None
+    user_id = stripe_session.metadata.get("user_id")
+    assert user_id is not None
+    user = get_active_user_profile_by_id_in_realm(user_id, session.customer.realm)
     payment_method = stripe_setup_intent.payment_method
-    assert isinstance(payment_method, (str, type(None)))
 
     if session.type in [
-        Session.CARD_UPDATE_FROM_BILLING_PAGE,
-        Session.CARD_UPDATE_FROM_UPGRADE_PAGE,
+        Session.UPGRADE_FROM_BILLING_PAGE,
+        Session.RETRY_UPGRADE_WITH_ANOTHER_PAYMENT_METHOD,
     ]:
-        billing_session.update_or_create_stripe_customer(payment_method)
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        assert session.payment_intent is not None
+        session.payment_intent.status = PaymentIntent.PROCESSING
+        session.payment_intent.last_payment_error = ()
+        session.payment_intent.save(update_fields=["status", "last_payment_error"])
+        try:
+            stripe.PaymentIntent.confirm(
+                session.payment_intent.stripe_payment_intent_id,
+                payment_method=payment_method,
+                off_session=True,
+            )
+        except stripe.error.CardError:
+            pass
+    elif session.type in [
+        Session.FREE_TRIAL_UPGRADE_FROM_BILLING_PAGE,
+        Session.FREE_TRIAL_UPGRADE_FROM_ONBOARDING_PAGE,
+    ]:
+        ensure_realm_does_not_have_active_plan(user.realm)
+        update_or_create_stripe_customer(user, payment_method)
+        process_initial_upgrade(
+            user,
+            int(stripe_setup_intent.metadata["licenses"]),
+            stripe_setup_intent.metadata["license_management"] == "automatic",
+            int(stripe_setup_intent.metadata["billing_schedule"]),
+            charge_automatically=True,
+            free_trial=True,
+        )
+    elif session.type in [Session.CARD_UPDATE_FROM_BILLING_PAGE]:
+        update_or_create_stripe_customer(user, payment_method)
 
 
 @error_handler
@@ -107,12 +119,13 @@ def handle_payment_intent_succeeded_event(
     payment_intent.status = PaymentIntent.SUCCEEDED
     payment_intent.save()
     metadata: Dict[str, Any] = stripe_payment_intent.metadata
+    assert payment_intent.customer.realm is not None
+    user_id = metadata.get("user_id")
+    assert user_id is not None
+    user = get_active_user_profile_by_id_in_realm(user_id, payment_intent.customer.realm)
 
     description = ""
-    charge: stripe.Charge
-    for charge in stripe_payment_intent.charges:  # type: ignore[attr-defined] # https://stripe.com/docs/upgrades#2022-11-15
-        assert charge.payment_method_details is not None
-        assert charge.payment_method_details.card is not None
+    for charge in stripe_payment_intent.charges:
         description = f"Payment (Card ending in {charge.payment_method_details.card.last4})"
         break
 
@@ -123,30 +136,48 @@ def handle_payment_intent_succeeded_event(
         description=description,
         discountable=False,
     )
-    billing_session = get_billing_session_for_stripe_webhook(
-        payment_intent.customer, metadata.get("user_id")
-    )
-    plan_tier = int(metadata["plan_tier"])
     try:
-        billing_session.ensure_current_plan_is_upgradable(payment_intent.customer, plan_tier)
-    except (UpgradeWithExistingPlanError, InvalidPlanUpgradeError) as e:
+        ensure_realm_does_not_have_active_plan(user.realm)
+    except UpgradeWithExistingPlanError as e:
         stripe_invoice = stripe.Invoice.create(
             auto_advance=True,
             collection_method="charge_automatically",
             customer=stripe_payment_intent.customer,
             days_until_due=None,
-            statement_descriptor=CustomerPlan.name_from_tier(plan_tier).replace("Zulip ", "")
-            + " Credit",
+            statement_descriptor="Zulip Cloud Standard Credit",
         )
         stripe.Invoice.finalize_invoice(stripe_invoice)
         raise e
 
-    billing_session.process_initial_upgrade(
-        plan_tier,
+    process_initial_upgrade(
+        user,
         int(metadata["licenses"]),
         metadata["license_management"] == "automatic",
         int(metadata["billing_schedule"]),
         True,
         False,
-        billing_session.get_remote_server_legacy_plan(payment_intent.customer),
     )
+
+
+@error_handler
+def handle_payment_intent_payment_failed_event(
+    stripe_payment_intent: stripe.PaymentIntent, payment_intent: PaymentIntent
+) -> None:
+    payment_intent.status = PaymentIntent.get_status_integer_from_status_text(
+        stripe_payment_intent.status
+    )
+    assert payment_intent.customer.realm is not None
+    billing_logger.info(
+        "Stripe payment intent failed: %s %s %s %s",
+        payment_intent.customer.realm.string_id,
+        stripe_payment_intent.last_payment_error.get("type"),
+        stripe_payment_intent.last_payment_error.get("code"),
+        stripe_payment_intent.last_payment_error.get("param"),
+    )
+    payment_intent.last_payment_error = {
+        "description": stripe_payment_intent.last_payment_error.get("type"),
+    }
+    payment_intent.last_payment_error["message"] = stripe_payment_intent.last_payment_error.get(
+        "message"
+    )
+    payment_intent.save(update_fields=["status", "last_payment_error"])

corporate/lib/support.py

@@ -1,147 +1,15 @@
-from dataclasses import dataclass
-from decimal import Decimal
-from typing import Optional, TypedDict
 from urllib.parse import urlencode, urljoin, urlunsplit
 
 from django.conf import settings
 from django.urls import reverse
-from django.utils.timezone import now as timezone_now
 
-from corporate.lib.stripe import BillingSession
-from corporate.models import (
-    Customer,
-    CustomerPlan,
-    ZulipSponsorshipRequest,
-    get_current_plan_by_customer,
-)
-from zerver.models import Realm, get_org_type_display_name, get_realm
-from zilencer.lib.remote_counts import MissingDataError
+from zerver.models import Realm, get_realm
 
 
-class SponsorshipRequestDict(TypedDict):
-    org_type: str
-    org_website: str
-    org_description: str
-    total_users: str
-    paid_users: str
-    paid_users_description: str
-    requested_plan: str
-
-
-@dataclass
-class SponsorshipData:
-    sponsorship_pending: bool = False
-    default_discount: Optional[Decimal] = None
-    latest_sponsorship_request: Optional[SponsorshipRequestDict] = None
-
-
-@dataclass
-class PlanData:
-    customer: Optional["Customer"] = None
-    current_plan: Optional["CustomerPlan"] = None
-    licenses: Optional[int] = None
-    licenses_used: Optional[int] = None
-    is_legacy_plan: bool = False
-    has_fixed_price: bool = False
-    warning: Optional[str] = None
-
-
-@dataclass
-class SupportData:
-    plan_data: PlanData
-    sponsorship_data: SponsorshipData
-
-
-def get_realm_support_url(realm: Realm) -> str:
+def get_support_url(realm: Realm) -> str:
     support_realm_uri = get_realm(settings.STAFF_SUBDOMAIN).uri
     support_url = urljoin(
         support_realm_uri,
         urlunsplit(("", "", reverse("support"), urlencode({"q": realm.string_id}), "")),
     )
     return support_url
-
-
-def get_customer_discount_for_support_view(
-    customer: Optional[Customer] = None,
-) -> Optional[Decimal]:
-    if customer is None:
-        return None
-    return customer.default_discount
-
-
-def get_customer_sponsorship_data(customer: Customer) -> SponsorshipData:
-    pending = customer.sponsorship_pending
-    discount = customer.default_discount
-    sponsorship_request = None
-    if pending:
-        last_sponsorship_request = (
-            ZulipSponsorshipRequest.objects.filter(customer=customer).order_by("id").last()
-        )
-        if last_sponsorship_request is not None:
-            org_type_name = get_org_type_display_name(last_sponsorship_request.org_type)
-            if (
-                last_sponsorship_request.org_website is None
-                or last_sponsorship_request.org_website == ""
-            ):
-                website = "No website submitted"
-            else:
-                website = last_sponsorship_request.org_website
-            sponsorship_request = SponsorshipRequestDict(
-                org_type=org_type_name,
-                org_website=website,
-                org_description=last_sponsorship_request.org_description,
-                total_users=last_sponsorship_request.expected_total_users,
-                paid_users=last_sponsorship_request.paid_users_count,
-                paid_users_description=last_sponsorship_request.paid_users_description,
-                requested_plan=last_sponsorship_request.requested_plan,
-            )
-
-    return SponsorshipData(
-        sponsorship_pending=pending,
-        default_discount=discount,
-        latest_sponsorship_request=sponsorship_request,
-    )
-
-
-def get_current_plan_data_for_support_view(billing_session: BillingSession) -> PlanData:
-    customer = billing_session.get_customer()
-    plan = None
-    if customer is not None:
-        plan = get_current_plan_by_customer(customer)
-    plan_data = PlanData(
-        customer=customer,
-        current_plan=plan,
-    )
-    if plan is not None:
-        new_plan, last_ledger_entry = billing_session.make_end_of_cycle_updates_if_needed(
-            plan, timezone_now()
-        )
-        if last_ledger_entry is not None:
-            if new_plan is not None:
-                plan_data.current_plan = new_plan  # nocoverage
-            plan_data.licenses = last_ledger_entry.licenses
-            try:
-                plan_data.licenses_used = billing_session.current_count_for_billed_licenses()
-            except MissingDataError:  # nocoverage
-                plan_data.warning = "Recent data missing: No information for used licenses"
-        assert plan_data.current_plan is not None  # for mypy
-        plan_data.is_legacy_plan = (
-            plan_data.current_plan.tier == CustomerPlan.TIER_SELF_HOSTED_LEGACY
-        )
-        plan_data.has_fixed_price = plan_data.current_plan.fixed_price is not None
-
-    return plan_data
-
-
-def get_data_for_support_view(billing_session: BillingSession) -> SupportData:
-    plan_data = get_current_plan_data_for_support_view(billing_session)
-    customer = billing_session.get_customer()
-    if customer is not None:
-        sponsorship_data = get_customer_sponsorship_data(customer)
-    else:
-        sponsorship_data = SponsorshipData()
-
-    return SupportData(
-        plan_data=plan_data,
-        sponsorship_data=sponsorship_data,
-    )

corporate/migrations/0001_initial.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     initial = True
 
     dependencies = [

corporate/migrations/0002_customer_default_discount.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0001_initial"),
     ]

corporate/migrations/0003_customerplan.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0002_customer_default_discount"),
     ]

corporate/migrations/0004_licenseledger.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0003_customerplan"),
     ]

corporate/migrations/0005_customerplan_invoicing.py

@@ -5,6 +5,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0004_licenseledger"),
     ]

corporate/migrations/0006_nullable_stripe_customer_id.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0005_customerplan_invoicing"),
     ]

corporate/migrations/0007_remove_deprecated_fields.py

@@ -4,6 +4,7 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0006_nullable_stripe_customer_id"),
     ]

corporate/migrations/0008_nullable_next_invoice_date.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0007_remove_deprecated_fields"),
     ]

corporate/migrations/0009_customer_sponsorship_pending.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0008_nullable_next_invoice_date"),
     ]

corporate/migrations/0010_customerplan_exempt_from_from_license_number_check.py

@@ -4,6 +4,7 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("corporate", "0009_customer_sponsorship_pending"),
     ]