Release Zulip Server 1.5.1.

Now that we no longer use node_modules at all in production (it's only
used to generate static assets), we don't include `node_modules` in
the production tarballs, and thus we shouldn't attempt to copy
`node_modules` out of the production tarballs when installing.

Fixes a regression introduced in
d71f2e7b9b.

.codecov.yml

@@ -1,12 +0,0 @@
-comment: off
-
-coverage:
-  status:
-    project:
-      default:
-        target: auto
-        # Codecov has the tendency to report a lot of false negatives,
-        # so we basically suppress comments completely.
-        threshold: 50%
-        base: auto
-    patch: off

.codespellignore

@@ -1,30 +0,0 @@
-te
-ans
-pullrequest
-ist
-cros
-wit
-nwe
-circularly
-ned
-ba
-ressemble
-ser
-sur
-hel
-fpr
-alls
-nd
-ot
-womens
-vise
-falsy
-ro
-derails
-forin
-uper
-slac
-couldn
-ges
-assertIn
-thirdparty

.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG

.editorconfig

@@ -3,23 +3,17 @@ root = true
 [*]
 end_of_line = lf
 charset = utf-8
-indent_size = 4
-indent_style = space
-insert_final_newline = true
 trim_trailing_whitespace = true
+insert_final_newline = true
 
-[[shell]]
+[*.{sh,py,js, json,yml,xml, css, md,markdown, handlebars,html}]
-binary_next_line = true
+indent_style = space
-switch_case_indent = true
+indent_size = 4
 
-[{*.{js,json,ts},check-openapi}]
+[*.{svg,rb,pp,pl}]
-max_line_length = 100
+indent_style = space
-
-[*.{py,pyi}]
-max_line_length = 110
-
-[*.{md,svg,rb,pp,yaml,yml}]
 indent_size = 2
 
-[package.json]
+[*.{cfg}]
-indent_size = 2
+indent_style = space
+indent_size = 8

.eslintignore

@@ -1,15 +1,3 @@
-# This is intended for generated files and vendored third-party files.
+static/js/bundle.js
-# For our source code, instead of adding files here, consider using
+static/js/blueslip.js
-# specific eslint-disable comments in the files themselves.
+puppet/zulip_ops/files/statsd/local.js
-
-/docs/_build
-/static/generated
-/static/webpack-bundles
-/var/*
-!/var/puppeteer
-/var/puppeteer/*
-!/var/puppeteer/test_credentials.d.ts
-/web/generated
-/web/third
-/zulip-current-venv
-/zulip-py3-venv

.eslintrc.js

@@ -1,293 +0,0 @@
-"use strict";
-
-const confusingBrowserGlobals = require("confusing-browser-globals");
-
-module.exports = {
-    root: true,
-    env: {
-        es2020: true,
-        node: true,
-    },
-    extends: [
-        "eslint:recommended",
-        "plugin:import/errors",
-        "plugin:import/warnings",
-        "plugin:no-jquery/recommended",
-        "plugin:no-jquery/deprecated",
-        "plugin:unicorn/recommended",
-        "prettier",
-    ],
-    parser: "@babel/eslint-parser",
-    parserOptions: {
-        requireConfigFile: false,
-        warnOnUnsupportedTypeScriptVersion: false,
-        sourceType: "unambiguous",
-    },
-    plugins: ["formatjs", "no-jquery"],
-    settings: {
-        formatjs: {
-            additionalFunctionNames: ["$t", "$t_html"],
-        },
-        "no-jquery": {
-            collectionReturningPlugins: {
-                expectOne: "always",
-            },
-            variablePattern: "^\\$(?!t$|t_html$).",
-        },
-    },
-    reportUnusedDisableDirectives: true,
-    rules: {
-        "array-callback-return": "error",
-        "arrow-body-style": "error",
-        "block-scoped-var": "error",
-        "consistent-return": "error",
-        curly: "error",
-        "dot-notation": "error",
-        eqeqeq: "error",
-        "formatjs/enforce-default-message": ["error", "literal"],
-        "formatjs/enforce-placeholders": [
-            "error",
-            {ignoreList: ["b", "code", "em", "i", "kbd", "p", "strong"]},
-        ],
-        "formatjs/no-id": "error",
-        "guard-for-in": "error",
-        "import/extensions": "error",
-        "import/first": "error",
-        "import/newline-after-import": "error",
-        "import/no-cycle": ["error", {ignoreExternal: true}],
-        "import/no-duplicates": "error",
-        "import/no-self-import": "error",
-        "import/no-unresolved": "off",
-        "import/no-useless-path-segments": "error",
-        "import/order": ["error", {alphabetize: {order: "asc"}, "newlines-between": "always"}],
-        "import/unambiguous": "error",
-        "lines-around-directive": "error",
-        "new-cap": "error",
-        "no-alert": "error",
-        "no-array-constructor": "error",
-        "no-bitwise": "error",
-        "no-caller": "error",
-        "no-catch-shadow": "error",
-        "no-constant-condition": ["error", {checkLoops: false}],
-        "no-div-regex": "error",
-        "no-else-return": "error",
-        "no-eq-null": "error",
-        "no-eval": "error",
-        "no-implicit-coercion": "error",
-        "no-implied-eval": "error",
-        "no-inner-declarations": "off",
-        "no-iterator": "error",
-        "no-jquery/no-append-html": "error",
-        "no-jquery/no-constructor-attributes": "error",
-        "no-jquery/no-parse-html-literal": "error",
-        "no-label-var": "error",
-        "no-labels": "error",
-        "no-loop-func": "error",
-        "no-multi-str": "error",
-        "no-native-reassign": "error",
-        "no-new-func": "error",
-        "no-new-object": "error",
-        "no-new-wrappers": "error",
-        "no-octal-escape": "error",
-        "no-plusplus": "error",
-        "no-proto": "error",
-        "no-restricted-globals": ["error", ...confusingBrowserGlobals],
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-self-compare": "error",
-        "no-sync": "error",
-        "no-throw-literal": "error",
-        "no-undef-init": "error",
-        "no-unneeded-ternary": ["error", {defaultAssignment: false}],
-        "no-unused-expressions": "error",
-        "no-unused-vars": [
-            "error",
-            {args: "all", argsIgnorePattern: "^_", ignoreRestSiblings: true},
-        ],
-        "no-use-before-define": ["error", {functions: false}],
-        "no-useless-concat": "error",
-        "no-useless-constructor": "error",
-        "no-var": "error",
-        "object-shorthand": ["error", "always", {avoidExplicitReturnArrows: true}],
-        "one-var": ["error", "never"],
-        "prefer-arrow-callback": "error",
-        "prefer-const": ["error", {ignoreReadBeforeAssign: true}],
-        radix: "error",
-        "sort-imports": ["error", {ignoreDeclarationSort: true}],
-        "spaced-comment": ["error", "always", {markers: ["/"]}],
-        strict: "error",
-        "unicorn/consistent-function-scoping": "off",
-        "unicorn/explicit-length-check": "off",
-        "unicorn/filename-case": "off",
-        "unicorn/no-await-expression-member": "off",
-        "unicorn/no-negated-condition": "off",
-        "unicorn/no-null": "off",
-        "unicorn/no-process-exit": "off",
-        "unicorn/no-useless-undefined": "off",
-        "unicorn/numeric-separators-style": "off",
-        "unicorn/prefer-module": "off",
-        "unicorn/prefer-node-protocol": "off",
-        "unicorn/prefer-string-raw": "off",
-        "unicorn/prefer-ternary": "off",
-        "unicorn/prefer-top-level-await": "off",
-        "unicorn/prevent-abbreviations": "off",
-        "unicorn/switch-case-braces": "off",
-        "valid-typeof": ["error", {requireStringLiterals: true}],
-        yoda: "error",
-    },
-    overrides: [
-        {
-            files: ["web/tests/**"],
-            rules: {
-                "no-jquery/no-selector-prop": "off",
-            },
-        },
-        {
-            files: ["web/e2e-tests/**"],
-            globals: {
-                zulip_test: false,
-            },
-        },
-        {
-            files: ["web/src/**"],
-            globals: {
-                StripeCheckout: false,
-            },
-        },
-        {
-            files: ["**/*.ts"],
-            extends: [
-                "plugin:@typescript-eslint/strict-type-checked",
-                "plugin:@typescript-eslint/stylistic-type-checked",
-                "plugin:import/typescript",
-            ],
-            parserOptions: {
-                project: "tsconfig.json",
-            },
-            settings: {
-                "import/resolver": {
-                    node: {
-                        extensions: [".ts", ".d.ts", ".js"], // https://github.com/import-js/eslint-plugin-import/issues/2267
-                    },
-                },
-            },
-            globals: {
-                JQuery: false,
-            },
-            rules: {
-                // Disable base rule to avoid conflict
-                "no-use-before-define": "off",
-
-                "@typescript-eslint/consistent-type-assertions": [
-                    "error",
-                    {assertionStyle: "never"},
-                ],
-                "@typescript-eslint/consistent-type-definitions": ["error", "type"],
-                "@typescript-eslint/consistent-type-imports": "error",
-                "@typescript-eslint/explicit-function-return-type": [
-                    "error",
-                    {allowExpressions: true},
-                ],
-                "@typescript-eslint/member-ordering": "error",
-                "@typescript-eslint/method-signature-style": "error",
-                "@typescript-eslint/no-non-null-assertion": "off",
-                "@typescript-eslint/no-unnecessary-condition": "off",
-                "@typescript-eslint/no-unnecessary-qualifier": "error",
-                "@typescript-eslint/no-unused-vars": [
-                    "error",
-                    {args: "all", argsIgnorePattern: "^_", ignoreRestSiblings: true},
-                ],
-                "@typescript-eslint/no-use-before-define": ["error", {functions: false}],
-                "@typescript-eslint/parameter-properties": "error",
-                "@typescript-eslint/promise-function-async": "error",
-                "@typescript-eslint/restrict-plus-operands": ["error", {}],
-                "@typescript-eslint/restrict-template-expressions": ["error", {}],
-                "no-undef": "error",
-            },
-        },
-        {
-            files: ["**/*.d.ts"],
-            rules: {
-                "import/unambiguous": "off",
-            },
-        },
-        {
-            files: ["web/e2e-tests/**", "web/tests/**"],
-            globals: {
-                CSS: false,
-                document: false,
-                navigator: false,
-                window: false,
-            },
-            rules: {
-                "formatjs/no-id": "off",
-                "new-cap": "off",
-                "no-sync": "off",
-                "unicorn/prefer-prototype-methods": "off",
-            },
-        },
-        {
-            files: ["web/debug-require.js"],
-            env: {
-                browser: true,
-                es2020: false,
-            },
-            rules: {
-                // Don’t require ES features that PhantomJS doesn’t support
-                // TODO: Toggle these settings now that we don't use PhantomJS
-                "no-var": "off",
-                "object-shorthand": "off",
-                "prefer-arrow-callback": "off",
-            },
-        },
-        {
-            files: ["web/shared/**", "web/src/**", "web/third/**"],
-            env: {
-                browser: true,
-                node: false,
-            },
-            globals: {
-                DEVELOPMENT: false,
-                ZULIP_VERSION: false,
-            },
-            rules: {
-                "no-console": "error",
-            },
-            settings: {
-                "import/resolver": {
-                    webpack: {
-                        config: "./web/webpack.config.ts",
-                    },
-                },
-            },
-        },
-        {
-            files: ["web/shared/**"],
-            env: {
-                browser: false,
-                "shared-node-browser": true,
-            },
-            rules: {
-                "import/no-restricted-paths": [
-                    "error",
-                    {
-                        zones: [
-                            {
-                                target: "./web/shared",
-                                from: ".",
-                                except: ["./node_modules", "./web/shared"],
-                            },
-                        ],
-                    },
-                ],
-                "unicorn/prefer-string-replace-all": "off",
-            },
-        },
-        {
-            files: ["web/server/**"],
-            env: {
-                node: true,
-            },
-        },
-    ],
-};

.eslintrc.json

@@ -0,0 +1,246 @@
+{
+    "env": {
+        "node": true
+    },
+    "globals": {
+        "$": false,
+        "_": false,
+        "jQuery": false,
+        "Spinner": false,
+        "Handlebars": false,
+        "XDate": false,
+        "zxcvbn": false,
+        "LazyLoad": false,
+        "Dropbox": false,
+        "SockJS": false,
+        "marked": false,
+        "i18n": false,
+        "bridge": false,
+        "page_params": false,
+        "status_classes": false,
+        "password_quality": false,
+        "csrf_token": false,
+        "typeahead_helper": false,
+        "popovers": false,
+        "server_events": false,
+        "ui": false,
+        "stream_color": false,
+        "people": false,
+        "navigate": false,
+        "settings": false,
+        "resize": false,
+        "loading": false,
+        "compose": false,
+        "compose_fade": false,
+        "subs": false,
+        "timerender": false,
+        "message_live_update": false,
+        "message_edit": false,
+        "reload": false,
+        "composebox_typeahead": false,
+        "search": false,
+        "topic_list": false,
+        "gear_menu": false,
+        "hashchange": false,
+        "message_list": false,
+        "Filter": false,
+        "pointer": false,
+        "util": false,
+        "MessageListView": false,
+        "blueslip": false,
+        "rows": false,
+        "WinChan": false,
+        "muting_ui": false,
+        "Socket": false,
+        "channel": false,
+        "components": false,
+        "viewport": false,
+        "avatar": false,
+        "feature_flags": false,
+        "search_suggestion": false,
+        "referral": false,
+        "notifications": false,
+        "message_flags": false,
+        "bot_data": false,
+        "stream_list": false,
+        "narrow": false,
+        "admin": false,
+        "stream_data": false,
+        "muting": false,
+        "Dict": false,
+        "unread": false,
+        "alert_words_ui": false,
+        "message_store": false,
+        "favicon": false,
+        "condense": false,
+        "floating_recipient_bar": false,
+        "tab_bar": false,
+        "emoji": false,
+        "activity": false,
+        "invite": false,
+        "colorspace": false,
+        "reactions": false,
+        "tutorial": false,
+        "templates": false,
+        "alert_words": false,
+        "fenced_code": false,
+        "echo": false,
+        "localstorage": false,
+        "current_msg_list": true,
+        "home_msg_list": false,
+        "pm_list": false,
+        "unread_ui": false,
+        "user_events": false,
+        "Plotly": false,
+        "emoji_codes": false
+    },
+    "rules": {
+        "no-restricted-syntax": 0,
+        "no-nested-ternary": 0,
+        "spaced-comment": 0,
+        "space-infix-ops": 0,
+        "newline-per-chained-call": 0,
+        "no-whitespace-before-property": 0,
+        "padded-blocks": 0,
+        "space-in-parens": 0,
+        "eol-last": ["error", "always"],
+        "no-unneeded-ternary": ["error", { "defaultAssignment": false }],
+        "no-case-declarations": "error",
+        "eqeqeq": ["error", "allow-null"],
+        "no-duplicate-imports": "error",
+        "no-undef": "error",
+        "dot-notation": ["error", { "allowKeywords": true }],
+        "no-iterator": "error",
+        "no-dupe-class-members": "error",
+        "no-useless-constructor": "error",
+        "prefer-const": ["error", {
+            "destructuring": "any",
+            "ignoreReadBeforeAssign": true
+        }],
+        "no-const-assign": "error",
+        "no-new-object": 2,
+        "quote-props": ["error", "as-needed", {
+            "keywords": false,
+            "unnecessary": true,
+            "numbers": false
+        }],
+        "no-array-constructor": "error",
+        "array-callback-return": "error",
+        "template-curly-spacing": "error",
+        //The Zulip codebase complies partially with the "no-useless-escape" rule; only regex expressions haven't been updated yet.
+        //Updated regex expressions are currently being tested in casper files and will decide about a potential future enforcement of this rule.
+        "no-useless-escape": 0,
+        "func-style": ["off", "expression"],
+        "wrap-iife": ["error", "outside", { "functionPrototypeMethods": false }],
+        "no-new-func": "error",
+        "space-before-function-paren": ["error", { "anonymous": "always", "named": "never", "asyncArrow": "always" }],
+        "no-param-reassign": 0,
+        "prefer-spread": "error",
+        "arrow-spacing": ["error", { "before": true, "after": true }],
+        "no-alert": 2,
+        "no-array-constructor": 2,
+        "no-caller": 2,
+        "no-bitwise": 2,
+        "no-catch-shadow": 2,
+        "comma-dangle": ["error", {
+            "arrays": "always-multiline",
+            "objects": "always-multiline",
+            "imports": "always-multiline",
+            "exports": "always-multiline",
+            "functions": "never"
+        }],
+        "no-console": 0,
+        "no-control-regex": 2,
+        "no-debugger": 2,
+        "no-div-regex": 2,
+        "no-dupe-keys": 2,
+        "no-else-return": 2,
+        "no-empty": 2,
+        "no-empty-character-class": 2,
+        "no-eq-null": 2,
+        "no-eval": 2,
+        "no-ex-assign": 2,
+        "no-extra-semi": 2,
+        "no-func-assign": 2,
+        "no-floating-decimal": 2,
+        "no-implied-eval": 2,
+        "no-with": 2,
+        "no-fallthrough": 2,
+        "no-unreachable": 2,
+        "no-undef": 2,
+        "no-undef-init": 2,
+        "no-unused-expressions": 2,
+        "no-octal": 2,
+        "no-octal-escape": 2,
+        "no-obj-calls": 2,
+        "no-multi-str": 2,
+        "no-new-wrappers": 2,
+        "no-new": 2,
+        "no-new-func": 2,
+        "no-native-reassign": 2,
+        "no-plusplus": 2,
+        "no-delete-var": 2,
+        "no-return-assign": 2,
+        "no-new-object": 2,
+        "no-label-var": 2,
+        "no-ternary": 0,
+        "no-self-compare": 2,
+        "no-sync": 2,
+        "no-underscore-dangle": 0,
+        "no-loop-func": 2,
+        "no-labels": 2,
+        "no-unused-vars": ["error", { "vars": "local", "args": "after-used",
+                                      "varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
+                                    }],
+        "no-script-url": 2,
+        "no-proto": 2,
+        "no-iterator": 2,
+        "no-mixed-requires": [0, false],
+        "no-extra-parens": ["error", "functions"],
+        "no-shadow": 0,
+        "no-use-before-define": 2,
+        "no-redeclare": 2,
+        "no-regex-spaces": 2,
+        "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
+        "block-scoped-var": 2,
+        "camelcase": 0,
+        "complexity": [0, 4],
+        "curly": 2,
+        "dot-notation": 2,
+        "guard-for-in": 2,
+        "max-depth": [0, 4],
+        "max-len": ["error", 100, 2, {
+            "ignoreUrls": true,
+            "ignoreComments": false,
+            "ignoreRegExpLiterals": true,
+            "ignoreStrings": true,
+            "ignoreTemplateLiterals": true
+        }],
+        "max-params": [0, 3],
+        "max-statements": [0, 10],
+        "new-cap": ["error", { "newIsCap": true, "capIsNew": false }],
+        "new-parens": 2,
+        "one-var": ["error", "never"],
+        "quotes": [0, "single"],
+        "quote-props": ["error", "as-needed", { "keywords": false, "unnecessary": true, "numbers": false }],
+        "radix": 2,
+        "semi": 2,
+        "keyword-spacing": ["error", {
+            "before": true,
+            "after": true,
+            "overrides": {
+                "return": { "after": true },
+                "throw": { "after": true },
+                "case": { "after": true }
+            }
+        }],
+        "space-before-blocks": 2,
+        "strict": 0,
+        "unnecessary-strict": 0,
+        "use-isnan": 2,
+        "valid-typeof": ["error", { "requireStringLiterals": true }],
+        "wrap-iife": 2,
+        "wrap-regex": 0,
+        "yoda": 2
+    }
+}

.gitattributes

@@ -1,32 +1,29 @@
-# DIFFS: Noise suppression.
-#
-# Suppress noisy generated files in diffs.
-# (When you actually want to see these diffs, use `git diff -a`.)
-
-# Large test fixtures:
-corporate/tests/stripe_fixtures/*.json -diff
-
-
-# FORMATTING
-
-# Maintain LF (Unix-style) newlines in text files.
 *   text=auto eol=lf
-
-# Make sure various media files never get somehow auto-detected as text
-# and then newline-converted.
 *.gif binary
 *.jpg binary
-*.jpeg binary
 *.eot binary
 *.woff binary
-*.woff2 binary
 *.svg binary
 *.ttf binary
 *.png binary
 *.otf binary
 *.tif binary
-*.ogg binary
+.gitignore export-ignore
-*.bson binary
+.gitattributes export-ignore
-*.bmp binary
+/static/assets export-ignore
-*.mp3 binary
+/bots export-ignore
-*.pdf binary
+/corporate export-ignore
+/static export-ignore
+/tools export-ignore
+/zilencer export-ignore
+/templates/corporate export-ignore
+/templates/zilencer export-ignore
+/puppet/zulip_internal export-ignore
+/zproject/local_settings.py export-ignore
+/zproject/test_settings.py export-ignore
+/zerver/fixtures export-ignore
+/zerver/tests export-ignore
+/frontend_tests export-ignore
+/node_modules export-ignore
+/humbug export-ignore
+/locale export-ignore

.github/FUNDING.yml

@@ -1,3 +0,0 @@
-github: zulip
-patreon: zulip
-open_collective: zulip

.github/ISSUE_TEMPLATE/1_discussed_on_czo.md

@@ -1,10 +0,0 @@
----
-name: Issue discussed in the Zulip development community
-about: Bug report, feature or improvement already discussed on chat.zulip.org.
----
-
-<!-- Issue description -->
-
-<!-- Link to a message in the chat.zulip.org discussion. Message links will still work even if the topic is renamed or resolved. Link back to this issue from the chat.zulip.org thread. -->
-
-CZO thread

.github/ISSUE_TEMPLATE/2_bug_report.md

@@ -1,18 +0,0 @@
----
-name: Bug report
-about: A concrete bug report with steps to reproduce the behavior. (See also "Possible bug" below.)
-labels: ["bug"]
----
-
-<!-- Describe what you were expecting to see, what you saw instead, and steps to take in order to reproduce the buggy behavior. Screenshots can be helpful. -->
-
-<!-- Check the box for the version of Zulip you are using (see https://zulip.com/help/view-zulip-version).-->
-
-**Zulip Server and web app version:**
-
-- [ ] Zulip Cloud (`*.zulipchat.com`)
-- [ ] Zulip Server 8.0+
-- [ ] Zulip Server 7.0+
-- [ ] Zulip Server 6.0+
-- [ ] Zulip Server 5.0 or older
-- [ ] Other or not sure

.github/ISSUE_TEMPLATE/3_feature_request.md

@@ -1,6 +0,0 @@
----
-name: Feature or improvement request
-about: A specific proposal for a new feature of improvement. (See also "Feature suggestion or feedback" below.)
----
-
-<!-- Describe the proposal, including how it would help you or your organization. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +0,0 @@
-blank_issues_enabled: true
-contact_links:
-  - name: Possible bug
-    url: https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html
-    about: Report unexpected behavior that may be a bug.
-  - name: Feature suggestion or feedback
-    url: https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html
-    about: Start a discussion about your idea for improving Zulip.
-  - name: Issue with running or upgrading a Zulip server
-    url: https://zulip.readthedocs.io/en/latest/production/troubleshooting.html
-    about: We provide free, interactive support for the vast majority of questions about running a Zulip server.
-  - name: Other support requests and sales questions
-    url: https://zulip.com/help/contact-support
-    about: Contact us — we're happy to help!

.github/pull_request_template.md

@@ -1,43 +0,0 @@
-<!-- Describe your pull request here.-->
-
-Fixes: <!-- Issue link, or clear description.-->
-
-<!-- If the PR makes UI changes, always include one or more still screenshots to demonstrate your changes. If it seems helpful, add a screen capture of the new functionality as well.
-
-Tooling tips: https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
--->
-
-**Screenshots and screen captures:**
-
-<details>
-<summary>Self-review checklist</summary>
-
-<!-- Prior to submitting a PR, follow our step-by-step guide to review your own code:
-https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code -->
-
-<!-- Once you create the PR, check off all the steps below that you have completed.
-If any of these steps are not relevant or you have not completed, leave them unchecked.-->
-
-- [ ] [Self-reviewed](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code) the changes for clarity and maintainability
-      (variable names, code reuse, readability, etc.).
-
-Communicate decisions, questions, and potential concerns.
-
-- [ ] Explains differences from previous plans (e.g., issue description).
-- [ ] Highlights technical choices and bugs encountered.
-- [ ] Calls out remaining decisions and concerns.
-- [ ] Automated tests verify logic where appropriate.
-
-Individual commits are ready for review (see [commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html)).
-
-- [ ] Each commit is a coherent idea.
-- [ ] Commit message(s) explain reasoning and motivation for changes.
-
-Completed manual review and testing of the following:
-
-- [ ] Visual appearance of the changes.
-- [ ] Responsiveness and internationalization.
-- [ ] Strings and tooltips.
-- [ ] End-to-end functionality of buttons, interactions and flows.
-- [ ] Corner cases, error conditions, and easily imagined bugs.
-</details>

.github/workflows/codeql-analysis.yml

@@ -1,40 +0,0 @@
-name: "Code scanning"
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-    branches: ["*.x", chat.zulip.org, main]
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  CodeQL:
-    permissions:
-      actions: read # for github/codeql-action/init to get workflow details
-      contents: read # for actions/checkout to fetch code
-      security-events: write # for github/codeql-action/analyze to upload SARIF results
-    if: ${{!github.event.repository.private}}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
-
-        # Override language selection by uncommenting this and choosing your languages
-        # with:
-        #   languages: go, javascript, csharp, python, cpp, java
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3

.github/workflows/production-suite.yml

@@ -1,307 +0,0 @@
-name: Zulip production suite
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-    paths:
-      - .github/workflows/production-suite.yml
-      - "**/migrations/**"
-      - manage.py
-      - pnpm-lock.yaml
-      - puppet/**
-      - requirements/**
-      - scripts/**
-      - tools/**
-      - web/babel.config.js
-      - web/postcss.config.js
-      - web/third/**
-      - web/webpack.config.ts
-      - zerver/worker/queue_processors.py
-      - zerver/lib/push_notifications.py
-      - zerver/decorator.py
-      - zproject/**
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-permissions:
-  contents: read
-
-jobs:
-  production_build:
-    # This job builds a release tarball from the current commit, which
-    # will be used for all of the following install/upgrade tests.
-    name: Ubuntu 22.04 production build
-    runs-on: ubuntu-latest
-
-    # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-    # the top explain how to build and upload these images.
-    # Ubuntu 22.04 ships with Python 3.10.12.
-    container: zulip/ci:jammy
-
-    steps:
-      - name: Add required permissions
-        run: |
-          # The checkout actions doesn't clone to ~/zulip or allow
-          # us to use the path option to clone outside the current
-          # /__w/zulip/zulip directory. Since this directory is owned
-          # by root we need to change it's ownership to allow the
-          # github user to clone the code here.
-          # Note: /__w/ is a docker volume mounted to $GITHUB_WORKSPACE
-          # which is /home/runner/work/.
-          sudo chown -R github .
-
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-      - uses: actions/checkout@v4
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore pnpm store
-        uses: actions/cache@v4
-        with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-jammy-${{ hashFiles('pnpm-lock.yaml') }}
-
-      - name: Restore python cache
-        uses: actions/cache@v4
-        with:
-          path: /srv/zulip-venv-cache
-          key: v1-venv-jammy-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-jammy
-
-      - name: Restore emoji cache
-        uses: actions/cache@v4
-        with:
-          path: /srv/zulip-emoji-cache
-          key: v1-emoji-jammy-${{ hashFiles('tools/setup/emoji/emoji_map.json') }}-${{ hashFiles('tools/setup/emoji/build_emoji') }}-${{ hashFiles('tools/setup/emoji/emoji_setup_utils.py') }}-${{ hashFiles('tools/setup/emoji/emoji_names.py') }}-${{ hashFiles('package.json') }}
-          restore-keys: v1-emoji-jammy
-
-      - name: Build production tarball
-        run: ./tools/ci/production-build
-
-      - name: Upload production build artifacts for install jobs
-        uses: actions/upload-artifact@v4
-        with:
-          name: production-tarball
-          path: /tmp/production-build
-          retention-days: 1
-
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
-
-  production_install:
-    # This job installs the server release tarball built above on a
-    # range of platforms, and does some basic health checks on the
-    # resulting installer Zulip server.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Docker images are built from 'tools/ci/Dockerfile'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 production install and PostgreSQL upgrade with pgroonga
-            os: jammy
-            extra-args: ""
-
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 production install
-            os: noble
-            extra-args: ""
-
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 production install with custom db name and user
-            os: bookworm
-            extra-args: --test-custom-db
-
-    name: ${{ matrix.name  }}
-    container:
-      image: ${{ matrix.docker_image }}
-      options: --init
-    runs-on: ubuntu-latest
-    needs: production_build
-
-    steps:
-      - name: Download built production tarball
-        uses: actions/download-artifact@v4
-        with:
-          name: production-tarball
-          path: /tmp
-
-      - name: Add required permissions and setup
-        run: |
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-          # Since actions/download-artifact@v4 loses all the permissions
-          # of the tarball uploaded by the upload artifact fix those.
-          chmod +x /tmp/production-upgrade-pg
-          chmod +x /tmp/production-pgroonga
-          chmod +x /tmp/production-install
-          chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Install production
-        run: sudo /tmp/production-install ${{ matrix.extra-args }}
-
-      - name: Verify install
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Install pgroonga
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-pgroonga
-
-      - name: Verify install after installing pgroonga
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Upgrade postgresql
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-upgrade-pg
-
-      - name: Verify install after upgrading postgresql
-        if: ${{ matrix.os == 'jammy' }}
-        run: sudo /tmp/production-verify ${{ matrix.extra-args }}
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}
-
-  production_upgrade:
-    # The production upgrade job starts with a container with a
-    # previous Zulip release installed, and attempts to upgrade it to
-    # the release tarball built for the current commit being tested.
-    #
-    # This is intended to catch bugs that result in the upgrade
-    # process failing.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Docker images are built from 'tools/ci/Dockerfile.prod'; the comments at
-          # the top explain how to build and upload these images.
-          - docker_image: zulip/ci:jammy-6.0
-            name: 6.0 Version Upgrade
-            os: jammy
-          - docker_image: zulip/ci:bookworm-7.0
-            name: 7.0 Version Upgrade
-            os: bookworm
-          - docker_image: zulip/ci:bookworm-8.0
-            name: 8.0 Version Upgrade
-            os: bookworm
-
-    name: ${{ matrix.name  }}
-    container:
-      image: ${{ matrix.docker_image }}
-      options: --init
-    runs-on: ubuntu-latest
-    needs: production_build
-
-    steps:
-      - name: Download built production tarball
-        uses: actions/download-artifact@v4
-        with:
-          name: production-tarball
-          path: /tmp
-
-      - name: Add required permissions and setup
-        run: |
-          # This is the GitHub Actions specific cache directory the
-          # the current github user must be able to access for the
-          # cache action to work. It is owned by root currently.
-          sudo chmod -R 0777 /__w/_temp/
-
-          # Since actions/download-artifact@v4 loses all the permissions
-          # of the tarball uploaded by the upload artifact fix those.
-          chmod +x /tmp/production-upgrade
-          chmod +x /tmp/production-verify
-          chmod +x /tmp/generate-failure-message
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Upgrade production
-        run: sudo /tmp/production-upgrade
-
-        # TODO: We should be running production-verify here, but it
-        # doesn't pass yet.
-        #
-        # - name: Verify install
-        #   run: sudo /tmp/production-verify
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: /tmp/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}

.github/workflows/update-oneclick-apps.yml

@@ -1,27 +0,0 @@
-name: Update one click apps
-on:
-  release:
-    types: [published]
-permissions:
-  contents: read
-
-jobs:
-  update-digitalocean-oneclick-app:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Update DigitalOcean one click app
-        env:
-          DIGITALOCEAN_API_KEY: ${{ secrets.ONE_CLICK_ACTION_DIGITALOCEAN_API_KEY }}
-          ZULIP_API_KEY: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_API_KEY }}
-          ZULIP_EMAIL: ${{ secrets.ONE_CLICK_ACTION_ZULIP_BOT_EMAIL }}
-          ZULIP_SITE: https://chat.zulip.org
-          ONE_CLICK_ACTION_STREAM: kandra ops
-          PYTHON_DIGITALOCEAN_REQUEST_TIMEOUT_SEC: 30
-          RELEASE_VERSION: ${{ github.event.release.tag_name }}
-        run: |
-          export PATH="$HOME/.local/bin:$PATH"
-          git clone https://github.com/zulip/marketplace-partners
-          pip3 install python-digitalocean zulip fab-classic PyNaCl
-          echo $PATH
-          python3 tools/oneclickapps/prepare_digital_ocean_one_click_app_release.py

.github/workflows/zulip-ci.yml

@@ -1,266 +0,0 @@
-# NOTE: Everything test in this file should be in `tools/test-all`.  If there's a
-# reason not to run it there, it should be there as a comment
-# explaining why.
-
-name: Zulip CI
-
-on:
-  push:
-    branches: ["*.x", chat.zulip.org, main]
-    tags: ["*"]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-permissions:
-  contents: read
-
-jobs:
-  tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # Base images are built using `tools/ci/Dockerfile.prod.template`.
-          # The comments at the top explain how to build and upload these images.
-          # Ubuntu 22.04 ships with Python 3.10.12.
-          - docker_image: zulip/ci:jammy
-            name: Ubuntu 22.04 (Python 3.10, backend + frontend)
-            os: jammy
-            include_documentation_tests: false
-            include_frontend_tests: true
-          # Debian 12 ships with Python 3.11.2.
-          - docker_image: zulip/ci:bookworm
-            name: Debian 12 (Python 3.11, backend + documentation)
-            os: bookworm
-            include_documentation_tests: true
-            include_frontend_tests: false
-          # Ubuntu 24.04 ships with Python 3.12.2.
-          - docker_image: zulip/ci:noble
-            name: Ubuntu 24.04 (Python 3.12, backend)
-            os: noble
-            include_documentation_tests: false
-            include_frontend_tests: false
-
-    runs-on: ubuntu-latest
-    name: ${{ matrix.name }}
-    container: ${{ matrix.docker_image }}
-    env:
-      # GitHub Actions sets HOME to /github/home which causes
-      # problem later in provision and frontend test that runs
-      # tools/setup/postgresql-init-dev-db because of the .pgpass
-      # location. PostgreSQL (psql) expects .pgpass to be at
-      # /home/github/.pgpass and setting home to `/home/github/`
-      # ensures it written there because we write it to ~/.pgpass.
-      HOME: /home/github/
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Create cache directories
-        run: |
-          dirs=(/srv/zulip-{venv,emoji}-cache)
-          sudo mkdir -p "${dirs[@]}"
-          sudo chown -R github "${dirs[@]}"
-
-      - name: Restore pnpm store
-        uses: actions/cache@v4
-        with:
-          path: /__w/.pnpm-store
-          key: v1-pnpm-store-${{ matrix.os }}-${{ hashFiles('pnpm-lock.yaml') }}
-
-      - name: Restore python cache
-        uses: actions/cache@v4
-        with:
-          path: /srv/zulip-venv-cache
-          key: v1-venv-${{ matrix.os }}-${{ hashFiles('requirements/dev.txt') }}
-          restore-keys: v1-venv-${{ matrix.os }}
-
-      - name: Restore emoji cache
-        uses: actions/cache@v4
-        with:
-          path: /srv/zulip-emoji-cache
-          key: v1-emoji-${{ matrix.os }}-${{ hashFiles('tools/setup/emoji/emoji_map.json', 'tools/setup/emoji/build_emoji', 'tools/setup/emoji/emoji_setup_utils.py', 'tools/setup/emoji/emoji_names.py', 'package.json') }}
-          restore-keys: v1-emoji-${{ matrix.os }}
-
-      - name: Install dependencies
-        run: |
-          # This is the main setup job for the test suite
-          ./tools/ci/setup-backend --skip-dev-db-build
-          scripts/lib/clean_unused_caches.py --verbose --threshold=0
-
-      - name: Run tools test
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-tools
-
-      - name: Run Codespell lint
-        run: |
-          source tools/ci/activate-venv
-          ./tools/run-codespell
-
-      # We run the tests that are only run in a specific job early, so
-      # that we get feedback to the developer about likely failures as
-      # quickly as possible. Backend/mypy failures that aren't
-      # identical across different versions are much more rare than
-      # frontend linter or node test failures.
-      - name: Run documentation and api tests
-        if: ${{ matrix.include_documentation_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # In CI, we only test links we control in test-documentation to avoid flakes
-          ./tools/test-documentation --skip-external-links
-          ./tools/test-help-documentation --skip-external-links
-          ./tools/test-api
-
-      - name: Run node tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Run the node tests first, since they're fast and deterministic
-          ./tools/test-js-with-node --coverage --parallel=1
-
-      - name: Run frontend lint
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/lint --groups=frontend --skip=gitlint # gitlint disabled because flaky
-
-      - name: Check schemas
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          # Check that various schemas are consistent. (is fast)
-          ./tools/check-schemas
-
-      - name: Check capitalization of strings
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./manage.py makemessages --locale en
-          PYTHONWARNINGS=ignore ./tools/check-capitalization --no-generate
-          PYTHONWARNINGS=ignore ./tools/check-frontend-i18n --no-generate
-
-      - name: Run puppeteer tests
-        if: ${{ matrix.include_frontend_tests }}
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-js-with-puppeteer
-
-      - name: Check pnpm dedupe
-        if: ${{ matrix.include_frontend_tests }}
-        run: pnpm dedupe --check
-
-      - name: Run backend lint
-        run: |
-          source tools/ci/activate-venv
-          echo "Test suite is running under $(python --version)."
-          ./tools/lint --groups=backend --skip=gitlint,mypy # gitlint disabled because flaky
-
-      - name: Run backend tests
-        run: |
-          source tools/ci/activate-venv
-          ./tools/test-backend ${{ matrix.os != 'bookworm' && '--coverage' || '' }} --xml-report --no-html-report --include-webhooks --include-transaction-tests --no-cov-cleanup --ban-console-output
-
-      - name: Run mypy
-        run: |
-          source tools/ci/activate-venv
-          # We run mypy after the backend tests so we get output from the
-          # backend tests, which tend to uncover more serious problems, first.
-          ./tools/run-mypy --version
-          ./tools/run-mypy
-
-      - name: Run miscellaneous tests
-        run: |
-          source tools/ci/activate-venv
-
-          # Currently our compiled requirements files will differ for different
-          # Python versions, so we will run test-locked-requirements only on the
-          # platform with the oldest one.
-          # ./tools/test-locked-requirements
-          # ./tools/test-run-dev  # https://github.com/zulip/zulip/pull/14233
-          #
-          # This test has been persistently flaky at like 1% frequency, is slow,
-          # and is for a very specific single feature, so we don't run it by default:
-          # ./tools/test-queue-worker-reload
-
-          ./tools/test-migrations
-          ./tools/setup/optimize-svg --check
-          ./tools/setup/generate_integration_bots_avatars.py --check-missing
-          ./tools/ci/check-executables
-
-          # Ban check-database-compatibility from transitively
-          # relying on static/generated, because it might not be
-          # up-to-date at that point in upgrade-zulip-stage-2.
-          chmod 000 static/generated web/generated
-          ./scripts/lib/check-database-compatibility
-          chmod 755 static/generated web/generated
-
-      - name: Check for untracked files
-        run: |
-          source tools/ci/activate-venv
-          # This final check looks for untracked files that may have been
-          # created by test-backend or provision.
-          untracked="$(git ls-files --exclude-standard --others)"
-          if [ -n "$untracked" ]; then
-              printf >&2 "Error: untracked files:\n%s\n" "$untracked"
-              exit 1
-          fi
-
-      - name: Test locked requirements
-        if: ${{ matrix.os == 'jammy' }}
-        run: |
-          . /srv/zulip-py3-venv/bin/activate && \
-          ./tools/test-locked-requirements
-
-      - name: Upload coverage reports
-
-        # Only upload coverage when both frontend and backend
-        # tests are run.
-        if: ${{ matrix.include_frontend_tests }}
-        uses: codecov/codecov-action@v4
-        with:
-          files: var/coverage.xml,var/node-coverage/lcov.info
-          token: ${{ secrets.CODECOV_TOKEN }}
-
-      - name: Store Puppeteer artifacts
-        # Upload these on failure, as well
-        if: ${{ always() && matrix.include_frontend_tests }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: puppeteer
-          path: ./var/puppeteer
-          retention-days: 60
-
-      - name: Check development database build
-        run: ./tools/ci/setup-backend
-
-      - name: Verify pnpm store path
-        run: |
-          set -x
-          path="$(pnpm store path)"
-          [[ "$path" == /__w/.pnpm-store/* ]]
-
-      - name: Generate failure report string
-        id: failure_report_string
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        run: tools/ci/generate-failure-message >> $GITHUB_OUTPUT
-
-      - name: Report status to CZO
-        if: ${{ failure() && github.repository == 'zulip/zulip' && github.event_name == 'push' }}
-        uses: zulip/github-actions-zulip/send-message@v1
-        with:
-          api-key: ${{ secrets.ZULIP_BOT_KEY }}
-          email: "github-actions-bot@chat.zulip.org"
-          organization-url: "https://chat.zulip.org"
-          to: "automated testing"
-          topic: ${{ steps.failure_report_string.outputs.topic }}
-          type: "stream"
-          content: ${{ steps.failure_report_string.outputs.content }}

.gitignore

@@ -1,95 +1,29 @@
-# Quick format and style primer:
-#
-#  * If a pattern is meant only for a specific location, it should have a
-#    leading slash, like `/staticfiles.json`.
-#    * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
-#      will do, but this makes a confusing pattern.  Adding a leading slash
-#      is clearer.
-#
-#  * Patterns like `.vscode/` without slashes, or with only a trailing slash,
-#    match in any subdirectory.
-#
-#  * Subdirectories with several internal things to ignore get their own
-#    `.gitignore` files.
-#
-#  * Comments must be on their own line.  (Otherwise they don't work.)
-#
-# See `git help ignore` for details on the format.
-
-## Config files for the dev environment
-/zproject/apns-dev.pem
-/zproject/apns-dev-key.p8
-/zproject/dev-secrets.conf
-/zproject/custom_dev_settings.py
-/tools/conf.ini
-/tools/custom_provision
-/tools/droplets/conf.ini
-
-## Byproducts of setting up and using the dev environment
 *.pyc
-*.tsbuildinfo
-package-lock.json
-
-/.vagrant
-/var/*
-!/var/puppeteer
-/var/puppeteer/*
-!/var/puppeteer/test_credentials.d.ts
-
-/.dmypy.json
-/.ruff_cache
-
-# Generated i18n data
-/locale/en
-/locale/language_options.json
-/locale/language_name_map.json
-/locale/*/mobile.json
-
-# Static build
-*.mo
-npm-debug.log
-/.pnpm-store
-/node_modules
-/prod-static
-/staticfiles.json
-/webpack-stats-production.json
-zulip-git-version
-
-# Test / analysis tools
-.coverage
-
-## Files (or really symlinks) created in a prod deployment
-/zproject/prod_settings.py
-/zulip-current-venv
-/zulip-py3-venv
-
-## Files left by various editors and local environments
-# (Ideally these should be in everyone's respective personal gitignore files.)
 *~
+/prod-static
 *.sw[po]
-.idea
+*.DS_Store
 .kdev4
+.idea
 zulip.kdev4
+coverage/
+.coverage
+/queue_error
 .kateproject.d/
 .kateproject
 *.kate-swp
 *.sublime-project
 *.sublime-workspace
-*.DS_Store
+.vagrant
-# VS Code. Avoid checking in .vscode in general, while still specifying
+/zproject/dev-secrets.conf
-# recommended extensions for working with this repository.
+static/js/bundle.js
-/.vscode/**/*
+static/generated/emoji
-!/.vscode/extensions.json
+static/generated/github-contributors.json
-# .cache/ is generated by VS Code test runner
+static/locale/language_options.json
-.cache/
+/node_modules
-.eslintcache
+/staticfiles.json
-
+npm-debug.log
-# Core dump files
+*.mo
-core
+var/*
-
+.vscode/
-# Static generated files for landing page.
+tools/conf.ini
-/static/images/landing-page/hello/generated
-
-## Miscellaneous
-# (Ideally this section is empty.)
-.transifexrc

.gitlint

@@ -1,13 +0,0 @@
-[general]
-ignore=title-trailing-punctuation, body-min-length, body-is-missing
-
-extra-path=tools/lib/gitlint_rules.py
-
-[title-match-regex]
-regex=^(.+:\ )?[A-Z].+\.$
-
-[title-max-length]
-line-length=72
-
-[body-max-line-length]
-line-length=76

.mailmap

@@ -1,164 +0,0 @@
-# This file teaches `git log` and friends the canonical names
-# and email addresses to use for our contributors.
-#
-# For details on the format, see:
-#   https://git.github.io/htmldocs/gitmailmap.html
-#
-# Handy commands for examining or adding to this file:
-#
-#     # shows all names/emails after mapping, sorted:
-#   $ git shortlog -es | sort -k2
-#
-#     # shows raw names/emails, filtered by mapped name:
-#   $ git log --format='%an %ae' --author=$NAME | uniq -c
-
-acrefoot <acrefoot@zulip.com> <acrefoot@alum.mit.edu>
-acrefoot <acrefoot@zulip.com> <acrefoot@dropbox.com>
-acrefoot <acrefoot@zulip.com> <acrefoot@humbughq.com>
-Adam Benesh <Adam.Benesh@gmail.com>
-Adam Benesh <Adam.Benesh@gmail.com> <Adam-Daniel.Benesh@t-systems.com>
-Adarsh Tiwari <xoldyckk@gmail.com>
-Aditya Chaudhary <aditya.chaudhary1558@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com>
-Adnan Shabbir Husain <generaladnan139@gmail.com> <78212328+adnan-td@users.noreply.github.com>
-Alex Vandiver <alexmv@zulip.com> <alex@chmrr.net>
-Alex Vandiver <alexmv@zulip.com> <github@chmrr.net>
-Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@humbughq.com>
-Allen Rabinovich <allenrabinovich@yahoo.com> <allenr@zulip.com>
-Alya Abbott <alya@zulip.com> <2090066+alya@users.noreply.github.com>
-Alya Abbott <alya@zulip.com> <alyaabbott@elance-odesk.com>
-Aman Agrawal <amanagr@zulip.com>
-Aman Agrawal <amanagr@zulip.com> <f2016561@pilani.bits-pilani.ac.in>
-Anders Kaseorg <anders@zulip.com> <anders@zulipchat.com>
-Anders Kaseorg <anders@zulip.com> <andersk@mit.edu>
-aparna-bhatt <aparnabhatt2001@gmail.com> <86338542+aparna-bhatt@users.noreply.github.com>
-Aryan Bhokare <aryan1bhokare@gmail.com>
-Aryan Bhokare <aryan1bhokare@gmail.com> <92683836+aryan-bhokare@users.noreply.github.com>
-Aryan Shridhar <aryanshridhar7@gmail.com>
-Aryan Shridhar <aryanshridhar7@gmail.com> <53977614+aryanshridhar@users.noreply.github.com>
-Ashwat Kumar Singh <ashwat.kumarsingh.met20@itbhu.ac.in>
-Austin Riba <austin@zulip.com> <austin@m51.io>
-Bedo Khaled <bedokhaled66@gmail.com>
-Bedo Khaled <bedokhaled66@gmail.com> <64221784+abdelrahman725@users.noreply.github.com>
-BIKI DAS <bikid475@gmail.com>
-Brijmohan Siyag <brijsiyag@gmail.com>
-Brock Whittaker <whittakerbrock@gmail.com> <bjwhitta@asu.edu>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.com>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulip.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brock@zulipchat.org>
-Brock Whittaker <whittakerbrock@gmail.com> <brockwhittaker@Brocks-MacBook.local>
-Chris Bobbe <cbobbe@zulip.com> <cbobbe@zulipchat.com>
-Chris Bobbe <cbobbe@zulip.com> <csbobbe@gmail.com>
-codewithnick <nikhilsingh526452@gmail.com>
-Danny Su <contact@dannysu.com> <opensource@emailengine.org>
-Dhruv Goyal <dhruvgoyal.dev@gmail.com>
-Dinesh <chdinesh1089@gmail.com>
-Dinesh <chdinesh1089@gmail.com> <chdinesh1089>
-Eeshan Garg <eeshan@zulip.com> <jerryguitarist@gmail.com>
-Eric Smith <erwsmith@gmail.com> <99841919+erwsmith@users.noreply.github.com>
-Evy Kassirer <evy@zulip.com>
-Evy Kassirer <evy@zulip.com> <evy.kassirer@gmail.com>
-Evy Kassirer <evy@zulip.com> <evykassirer@users.noreply.github.com>
-Ganesh Pawar <pawarg256@gmail.com> <58626718+ganpa3@users.noreply.github.com>
-Greg Price <greg@zulip.com> <gnprice@gmail.com>
-Greg Price <greg@zulip.com> <greg@zulipchat.com>
-Greg Price <greg@zulip.com> <price@mit.edu>
-Hardik Dharmani <Ddharmani99@gmail.com> <ddharmani99@gmail.com>
-Hemant Umre <hemantumre12@gmail.com> <87542880+HemantUmre12@users.noreply.github.com>
-Jai soni <jai_s@me.iitr.ac.in>
-Jai soni <jai_s@me.iitr.ac.in> <76561593+jai2201@users.noreply.github.com>
-Jeff Arnold <jbarnold@gmail.com> <jbarnold@humbughq.com>
-Jeff Arnold <jbarnold@gmail.com> <jbarnold@zulip.com>
-Jessica McKellar <jesstess@mit.edu> <jesstess@humbughq.com>
-Jessica McKellar <jesstess@mit.edu> <jesstess@zulip.com>
-John Lu <JohnLu10212004@gmail.com>
-John Lu <JohnLu10212004@gmail.com> <87673068+JohnLu2004@users.noreply.github.com>
-Joseph Ho <josephho678@gmail.com>
-Joseph Ho <josephho678@gmail.com> <62449508+Joelute@users.noreply.github.com>
-Julia Bichler <julia.bichler@tum.de> <74348920+juliaBichler01@users.noreply.github.com>
-Karl Stolley <karl@zulip.com> <karl@stolley.dev>
-Kevin Mehall <km@kevinmehall.net> <kevin@humbughq.com>
-Kevin Mehall <km@kevinmehall.net> <kevin@zulip.com>
-Kevin Scott <kevin.scott.98@gmail.com>
-Kislay Verma <kislayuv27@gmail.com>
-Kunal Sharma <v.shm.kunal@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com>
-Lalit Kumar Singh <lalitkumarsingh3716@gmail.com> <lalits01@smartek21.com>
-Lauryn Menard <lauryn@zulip.com> <63245456+laurynmm@users.noreply.github.com>
-Lauryn Menard <lauryn@zulip.com> <lauryn.menard@gmail.com>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in>
-m-e-l-u-h-a-n <purushottam.tiwari.cd.cse19@itbhu.ac.in> <pururshottam.tiwari.cd.cse19@itbhu.ac.in>
-Mateusz Mandera <mateusz.mandera@zulip.com> <mateusz.mandera@protonmail.com>
-Matt Keller <matt@zulip.com>
-Matt Keller <matt@zulip.com> <m@cognusion.com>
-Nehal Sharma <bablinaneh@gmail.com>
-Nehal Sharma <bablinaneh@gmail.com> <68962290+N-Shar-ma@users.noreply.github.com>
-Nimish Medatwal <medatwalnimish@gmail.com>
-Noble Mittal <noblemittal@outlook.com> <62551163+beingnoble03@users.noreply.github.com>
-nzai <nzaih18@gmail.com> <70953556+nzaih1999@users.noreply.github.com>
-Palash Baderia <palash.baderia@outlook.com>
-Palash Baderia <palash.baderia@outlook.com> <66828942+palashb01@users.noreply.github.com>
-Palash Raghuwanshi <singhpalash0@gmail.com>
-Parth <mittalparth22@gmail.com>
-Pratik Chanda <pratikchanda2000@gmail.com>
-Pratik Solanki <pratiksolanki2021@gmail.com>
-Priyam Seth <sethpriyam1@gmail.com> <b19188@students.iitmandi.ac.in>
-Ray Kraesig <rkraesig@zulip.com> <rkraesig@zulipchat.com>
-Reid Barton <rwbarton@gmail.com> <rwbarton@humbughq.com>
-Rein Zustand (rht) <rhtbot@protonmail.com>
-Rishabh Maheshwari <b20063@students.iitmandi.ac.in>
-Rishi Gupta <rishig@zulipchat.com> <rishig+git@mit.edu>
-Rishi Gupta <rishig@zulipchat.com> <rishig@kandralabs.com>
-Rishi Gupta <rishig@zulipchat.com> <rishig@users.noreply.github.com>
-Rixant Rokaha <rixantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rishantrokaha@gmail.com>
-Rixant Rokaha <rixantrokaha@gmail.com> <rrokaha@caldwell.edu>
-Rohan Gudimetla <rohan.gudimetla07@gmail.com>
-Sahil Batra <sahil@zulip.com> <35494118+sahil839@users.noreply.github.com>
-Sahil Batra <sahil@zulip.com> <sahilbatra839@gmail.com>
-Sanchit Sharma <ssharmas10662@gmail.com>
-Satyam Bansal <sbansal1999@gmail.com>
-Sayam Samal <samal.sayam@gmail.com>
-Scott Feeney <scott@oceanbase.org> <scott@humbughq.com>
-Scott Feeney <scott@oceanbase.org> <scott@zulip.com>
-Shashank Singh <21bec103@iiitdmj.ac.in>
-Shlok Patel <shlokcpatel2001@gmail.com>
-Shu Chen <shu@zulip.com>
-Shubham Padia <shubham@zulip.com>
-Shubham Padia <shubham@zulip.com> <shubham@glints.com>
-Somesh Ranjan <somesh.ranjan.met20@itbhu.ac.in> <77766761+somesh202@users.noreply.github.com>
-Steve Howell <showell@zulip.com> <showell30@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@yahoo.com>
-Steve Howell <showell@zulip.com> <showell@zulipchat.com>
-Steve Howell <showell@zulip.com> <steve@humbughq.com>
-Steve Howell <showell@zulip.com> <steve@zulip.com>
-strifel <info@strifel.de>
-Sujal Shah <sujalshah28092004@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com>
-Tanmay Kumar <tnmdotkr@gmail.com> <133781250+tnmkr@users.noreply.github.com>
-Tim Abbott <tabbott@zulip.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@dropbox.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@humbughq.com>
-Tim Abbott <tabbott@zulip.com> <tabbott@mit.edu>
-Tim Abbott <tabbott@zulip.com> <tabbott@zulipchat.com>
-Tomasz Kolek <tomasz-kolek@o2.pl> <tomasz-kolek@go2.pl>
-Ujjawal Modi <umodi2003@gmail.com> <99073049+Ujjawal3@users.noreply.github.com>
-umkay <ukhan@zulipchat.com> <umaimah.k@gmail.com>
-umkay <ukhan@zulipchat.com> <umkay@users.noreply.github.com>
-Viktor Illmer <1476338+v-ji@users.noreply.github.com>
-Vishnu KS <vishnu@zulip.com> <hackerkid@vishnuks.com>
-Vishnu KS <vishnu@zulip.com> <yo@vishnuks.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@dropbox.com>
-Waseem Daher <wdaher@zulip.com> <wdaher@humbughq.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com>
-Yash RE <33805964+YashRE42@users.noreply.github.com> <YashRE42@github.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com>
-Yogesh Sirsat <yogeshsirsat56@gmail.com> <41695888+yogesh-sirsat@users.noreply.github.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com>
-Zeeshan Equbal <equbalzeeshan@gmail.com> <54993043+zee-bit@users.noreply.github.com>
-Zev Benjamin <zev@zulip.com> <zev@dropbox.com>
-Zev Benjamin <zev@zulip.com> <zev@humbughq.com>
-Zev Benjamin <zev@zulip.com> <zev@mit.edu>
-Zixuan James Li <p359101898@gmail.com>
-Zixuan James Li <p359101898@gmail.com> <359101898@qq.com>
-Zixuan James Li <p359101898@gmail.com> <39874143+PIG208@users.noreply.github.com>

.npmrc

@@ -1 +0,0 @@
-ignore-dep-scripts=true

.prettierignore

@@ -1,17 +0,0 @@
-pnpm-lock.yaml
-/api_docs/**/*.md
-/corporate/tests/stripe_fixtures
-/help/**/*.md
-/locale
-/templates/**/*.md
-/tools/setup/emoji/emoji_map.json
-/web/third/*
-!/web/third/marked
-/web/third/marked/*
-!/web/third/marked/lib
-/web/third/marked/lib/*
-!/web/third/marked/lib/marked.d.ts
-/zerver/tests/fixtures
-/zerver/webhooks/*/doc.md
-/zerver/webhooks/github/githubsponsors.md
-/zerver/webhooks/*/fixtures

.pyre_configuration

@@ -1,15 +0,0 @@
-{
-  "source_directories": ["."],
-  "taint_models_path": [
-      "stubs/taint",
-      "zulip-py3-venv/lib/pyre_check/taint/"
-  ],
-  "search_path": [
-      "stubs/",
-      "zulip-py3-venv/lib/pyre_check/stubs/"
-  ],
-  "typeshed": "zulip-py3-venv/lib/pyre_check/typeshed/",
-  "exclude": [
-      "/srv/zulip/zulip-py3-venv/.*"
-  ]
-}

.readthedocs.yaml

@@ -1,15 +0,0 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/conf.py
-  fail_on_warning: true
-
-python:
-  install:
-    - requirements: requirements/docs.txt

.sonarcloud.properties

@@ -1 +0,0 @@
-sonar.inclusions=**/*.py,**/*.html

.travis.yml

@@ -0,0 +1,54 @@
+dist: trusty
+before_install:
+   - nvm install 0.10
+install:
+  # Disable Travis CI's built-in NVM installation
+  - mv ~/.nvm ~/.travis-nvm-disabled
+  - pip install coveralls
+  - tools/travis/setup-$TEST_SUITE
+  - tools/clean-venv-cache --travis
+cache:
+  - apt: false
+  - directories:
+    - $HOME/zulip-venv-cache
+    - node_modules
+    - $HOME/node
+env:
+  global:
+    - COVERAGE_FILE=var/.coverage
+    - COVERALLS_PARALLEL=true
+    - COVERALLS_SERVICE_NAME=travis-pro
+    - COVERALLS_REPO_TOKEN=hnXUEBKsORKHc8xIENGs9JjktlTb2HKlG
+    - BOTO_CONFIG=/tmp/nowhere
+  matrix:
+    - TEST_SUITE=frontend
+    - TEST_SUITE=backend
+language: python
+python:
+  - "2.7"
+  - "3.4"
+matrix:
+  include:
+    - python: "3.4"
+      env: TEST_SUITE=static-analysis
+    - python: "3.4"
+      env: TEST_SUITE=production
+    - python: "2.7"
+      env: TEST_SUITE=production
+# command to run tests
+script:
+  - unset GEM_PATH
+  - ./tools/travis/$TEST_SUITE
+sudo: required
+services:
+- docker
+addons:
+  artifacts:
+    paths:
+      - $(ls var/casper/* | tr "\n" ":")
+      - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
+  postgresql: "9.3"
+after_success:
+  coveralls
+notifications:
+  webhooks: https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN

.tx/config

@@ -1,40 +1,14 @@
-# Migrated from transifex-client format with `tx migrate`
-#
-# See https://developers.transifex.com/docs/using-the-client which hints at
-# this format, but in general, the headings are in the format of:
-#
-# [o:<org>:p:<project>:r:<resource>]
-
 [main]
 host = https://www.transifex.com
-lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
 
-[o:zulip:p:zulip:r:djangopo]
+[zulip.djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
+source_file = static/locale/en/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
 source_lang = en
 type = PO
+file_filter = static/locale/<lang>/LC_MESSAGES/django.po
 
-[o:zulip:p:zulip:r:mobile]
+[zulip.translationsjson]
-file_filter = locale/<lang>/mobile.json
+source_file = static/locale/en/translations.json
-source_file = locale/en/mobile.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
-source_lang = en
-type = KEYVALUEJSON
-
-[o:zulip:p:zulip-test:r:djangopo]
-file_filter = locale/<lang>/LC_MESSAGES/django.po
-source_file = locale/en/LC_MESSAGES/django.po
-source_lang = en
-type = PO
-
-[o:zulip:p:zulip-test:r:translationsjson]
-file_filter = locale/<lang>/translations.json
-source_file = locale/en/translations.json
 source_lang = en
 type = KEYVALUEJSON
+file_filter = static/locale/<lang>/translations.json

.vscode/extensions.json

@@ -1,23 +0,0 @@
-{
-    // Recommended VS Code extensions for zulip/zulip.
-    //
-    // VS Code prompts a user to install the recommended extensions
-    // when a workspace is opened for the first time.  The user can
-    // also review the list with the 'Extensions: Show Recommended
-    // Extensions' command.  See
-    // https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions
-    // for more information.
-    //
-    // Extension identifier format: ${publisher}.${name}.
-    // Example: vscode.csharp
-
-    "recommendations": [
-        "42crunch.vscode-openapi",
-        "dbaeumer.vscode-eslint",
-        "esbenp.prettier-vscode",
-        "ms-vscode-remote.vscode-remote-extensionpack"
-    ],
-
-    // Extensions recommended by VS Code which are not recommended for users of zulip/zulip.
-    "unwantedRecommendations": []
-}

CODE_OF_CONDUCT.md

@@ -1,173 +0,0 @@
-# Zulip Code of Conduct
-
-Like the technical community as a whole, the Zulip team and community is
-made up of a mixture of professionals and volunteers from all over the
-world, working on every aspect of the mission, including mentorship,
-teaching, and connecting people.
-
-Diversity is one of our huge strengths, but it can also lead to
-communication issues and unhappiness. To that end, we have a few ground
-rules that we ask people to adhere to. This code applies equally to
-founders, mentors, and those seeking help and guidance.
-
-This isn't an exhaustive list of things that you can't do. Rather, take it
-in the spirit in which it's intended --- a guide to make it easier to enrich
-all of us and the technical communities in which we participate.
-
-## Expected behavior
-
-The following behaviors are expected and requested of all community members:
-
-- Participate. In doing so, you contribute to the health and longevity of
-  the community.
-- Exercise consideration and respect in your speech and actions.
-- Attempt collaboration before conflict. Assume good faith.
-- Refrain from demeaning, discriminatory, or harassing behavior and speech.
-- Take action or alert community leaders if you notice a dangerous
-  situation, someone in distress, or violations of this code, even if they
-  seem inconsequential.
-- Community event venues may be shared with members of the public; be
-  respectful to all patrons of these locations.
-
-## Unacceptable behavior
-
-The following behaviors are considered harassment and are unacceptable
-within the Zulip community:
-
-- Jokes or derogatory language that singles out members of any race,
-  ethnicity, culture, national origin, color, immigration status, social and
-  economic class, educational level, language proficiency, sex, sexual
-  orientation, gender identity and expression, age, size, family status,
-  political belief, religion, and mental and physical ability.
-- Violence, threats of violence, or violent language directed against
-  another person.
-- Disseminating or threatening to disseminate another person's personal
-  information.
-- Personal insults of any sort.
-- Posting or displaying sexually explicit or violent material.
-- Inappropriate photography or recording.
-- Deliberate intimidation, stalking, or following (online or in person).
-- Unwelcome sexual attention. This includes sexualized comments or jokes,
-  inappropriate touching or groping, and unwelcomed sexual advances.
-- Sustained disruption of community events, including talks and
-  presentations.
-- Advocating for, or encouraging, any of the behaviors above.
-
-## Reporting and enforcement
-
-Harassment and other code of conduct violations reduce the value of the
-community for everyone. If someone makes you or anyone else feel unsafe or
-unwelcome, please report it to the community organizers at
-zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
-report either personally or anonymously.
-
-If a community member engages in unacceptable behavior, the community
-organizers may take any action they deem appropriate, up to and including a
-temporary ban or permanent expulsion from the community without warning (and
-without refund in the case of a paid event).
-
-If someone outside the development community (e.g., a user of the Zulip
-software) engages in unacceptable behavior that affects someone in the
-community, we still want to know. Even if we don't have direct control over
-the violator, the community organizers can still support the people
-affected, reduce the chance of a similar violation in the future, and take
-any direct action we can.
-
-The nature of reporting means it can only help after the fact. If you see
-something you can do while a violation is happening, do it. A lot of the
-harms of harassment and other violations can be mitigated by the victim
-knowing that the other people present are on their side.
-
-All reports will be kept confidential. In some cases, we may determine that a
-public statement will need to be made. In such cases, the identities of all
-victims and reporters will remain confidential unless those individuals
-instruct us otherwise.
-
-## Scope
-
-We expect all community participants (contributors, paid or otherwise,
-sponsors, and other guests) to abide by this Code of Conduct in all
-community venues, online and in-person, as well as in all private
-communications pertaining to community business.
-
-This Code of Conduct and its related procedures also applies to unacceptable
-behavior occurring outside the scope of community activities when such
-behavior has the potential to adversely affect the safety and well-being of
-community members.
-
-## License and attribution
-
-This Code of Conduct is adapted from the
-[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
-under a
-[Creative Commons BY-SA](https://creativecommons.org/licenses/by-sa/4.0/)
-license.
-
-## Moderating the Zulip community
-
-Anyone can help moderate the Zulip community by helping make sure that folks are
-aware of the [community guidelines](https://zulip.com/development-community/)
-and this Code of Conduct, and that we maintain a positive and respectful
-atmosphere.
-
-Here are some guidelines for you how can help:
-
-- Be friendly! Welcoming folks, thanking them for their feedback, ideas and effort,
-  and just trying to keep the atmosphere warm make the whole community function
-  more smoothly. New participants who feel accepted, listened to and respected
-  are likely to treat others the same way.
-
-- Be familiar with the [community
-  guidelines](https://zulip.com/development-community/), and cite them liberally
-  when a user violates them. Be polite but firm. Some examples:
-
-  - @user please note that there is no need to @-mention @\_**Tim Abbott** when
-    you ask a question. As noted in the [guidelines for this
-    community](https://zulip.com/development-community/):
-
-    > Use @-mentions sparingly… there is generally no need to @-mention a
-    > core contributor unless you need their timely attention.
-
-  - @user, please keep in mind the following [community
-    guideline](https://zulip.com/development-community/):
-
-    > Don’t ask the same question in multiple places. Moderators read every
-    > public stream, and make sure every question gets a reply.
-
-    I’ve gone ahead and moved the other copy of this message to this thread.
-
-  - If asked a question in a direct message that is better discussed in a public
-    stream:
-    > Hi @user! Please start by reviewing
-    > https://zulip.com/development-community/#community-norms to learn how to
-    > get help in this community.
-
-- Users sometimes think chat.zulip.org is a testing instance. When this happens,
-  kindly direct them to use the **#test here** stream.
-
-- If you see a message that’s posted in the wrong place, go ahead and move it if
-  you have permissions to do so, even if you don’t plan to respond to it.
-  Leaving the “Send automated notice to new topic” option enabled helps make it
-  clear what happened to the person who sent the message.
-
-  If you are responding to a message that's been moved, mention the user in your
-  reply, so that the mention serves as a notification of the new location for
-  their conversation.
-
-- If a user is posting spam, please report it to an administrator. They will:
-
-  - Change the user's name to `<name> (spammer)` and deactivate them.
-  - Delete any spam messages they posted in public streams.
-
-- We care very much about maintaining a respectful tone in our community. If you
-  see someone being mean or rude, point out that their tone is inappropriate,
-  and ask them to communicate their perspective in a respectful way in the
-  future. If you don’t feel comfortable doing so yourself, feel free to ask a
-  member of Zulip's core team to take care of the situation.
-
-- Try to assume the best intentions from others (given the range of
-  possibilities presented by their visible behavior), and stick with a friendly
-  and positive tone even when someone’s behavior is poor or disrespectful.
-  Everyone has bad days and stressful situations that can result in them
-  behaving not their best, and while we should be firm about our community
-  rules, we should also enforce them with kindness.

CONTRIBUTING.md

@@ -1,429 +0,0 @@
-# Contributing guide
-
-Welcome to the Zulip community!
-
-## Zulip development community
-
-The primary communication forum for the Zulip community is the Zulip
-server hosted at [chat.zulip.org](https://chat.zulip.org/):
-
-- **Users** and **administrators** of Zulip organizations stop by to
-  ask questions, offer feedback, and participate in product design
-  discussions.
-- **Contributors to the project**, including the **core Zulip
-  development team**, discuss ongoing and future projects, brainstorm
-  ideas, and generally help each other out.
-
-Everyone is welcome to [sign up](https://chat.zulip.org/) and
-participate — we love hearing from our users! Public streams in the
-community receive thousands of messages a week. We recommend signing
-up using the special invite links for
-[users](https://chat.zulip.org/join/t5crtoe62bpcxyisiyglmtvb/),
-[self-hosters](https://chat.zulip.org/join/wnhv3jzm6afa4raenedanfno/)
-and
-[contributors](https://chat.zulip.org/join/npzwak7vpmaknrhxthna3c7p/)
-to get a curated list of initial stream subscriptions.
-
-To learn how to get started participating in the community, including [community
-norms](https://zulip.com/development-community/#community-norms) and [where to
-post](https://zulip.com/development-community/#where-do-i-send-my-message),
-check out our [Zulip development community
-guide](https://zulip.com/development-community/). The Zulip community is
-governed by a [code of
-conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
-
-## Ways to contribute
-
-To make a code or documentation contribution, read our
-[step-by-step guide](#your-first-codebase-contribution) to getting
-started with the Zulip codebase. A small sample of the type of work that
-needs doing:
-
-- Bug squashing and feature development on our Python/Django
-  [backend](https://github.com/zulip/zulip), web
-  [frontend](https://github.com/zulip/zulip),
-  Flutter [mobile app](https://github.com/zulip/zulip-flutter) in beta,
-  or Electron [desktop app](https://github.com/zulip/zulip-desktop).
-- Building out our
-  [Python API and bots](https://github.com/zulip/python-zulip-api) framework.
-- [Writing an integration](https://zulip.com/api/integrations-overview).
-- Improving our [user](https://zulip.com/help/) or
-  [developer](https://zulip.readthedocs.io/en/latest/) documentation.
-- [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
-  and manually testing pull requests.
-
-**Non-code contributions**: Some of the most valuable ways to contribute
-don't require touching the codebase at all. For example, you can:
-
-- Report issues, including both [feature
-  requests](https://zulip.readthedocs.io/en/latest/contributing/suggesting-features.html)
-  and [bug
-  reports](https://zulip.readthedocs.io/en/latest/contributing/reporting-bugs.html).
-- [Give feedback](#user-feedback) if you are evaluating or using Zulip.
-- [Participate
-  thoughtfully](https://zulip.readthedocs.io/en/latest/contributing/design-discussions.html)
-  in design discussions.
-- [Sponsor Zulip](https://github.com/sponsors/zulip) through the GitHub sponsors program.
-- [Translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
-  Zulip into your language.
-- [Stay connected](#stay-connected) with Zulip, and [help others
-  find us](#help-others-find-zulip).
-
-## Your first codebase contribution
-
-This section has a step by step guide to starting as a Zulip codebase
-contributor. It's long, but don't worry about doing all the steps perfectly;
-no one gets it right the first time, and there are a lot of people available
-to help.
-
-- First, make an account on the
-  [Zulip community server](https://zulip.com/development-community/),
-  paying special attention to the
-  [community norms](https://zulip.com/development-community/#community-norms).
-  If you'd like, introduce yourself in
-  [#new members](https://chat.zulip.org/#narrow/channel/95-new-members), using
-  your name as the topic. Bonus: tell us about your first impressions of
-  Zulip, and anything that felt confusing/broken or interesting/helpful as you
-  started using the product.
-
-- Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
-
-- Set up the development environment for the Zulip codebase you want
-  to work on, and start getting familiar with the code.
-
-  - For the server and web app:
-
-    - [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
-      getting help in
-      [#provision help](https://chat.zulip.org/#narrow/channel/21-provision-help)
-      if you run into any troubles.
-    - Familiarize yourself with [using the development environment](https://zulip.readthedocs.io/en/latest/development/using.html).
-    - Go through the [new application feature
-      tutorial](https://zulip.readthedocs.io/en/latest/tutorials/new-feature-tutorial.html) to get familiar with
-      how the Zulip codebase is organized and how to find code in it.
-
-  - For the upcoming Flutter-based mobile app:
-    - Set up a development environment following the instructions in
-      [the project README](https://github.com/zulip/zulip-flutter).
-    - Start reading recent commits to see the code we're writing.
-      Use either a [graphical Git viewer][] like `gitk`, or `git log -p`
-      with [the "secret" to reading its output][git-log-secret].
-    - Pick some of the code that appears in those Git commits and
-      that looks interesting. Use your IDE to visit that code
-      and to navigate to related code, reading to see how it works
-      and how the codebase is organized.
-
-- Read the [Zulip guide to
-  Git](https://zulip.readthedocs.io/en/latest/git/index.html) if you
-  are unfamiliar with Git or Zulip's rebase-based Git workflow,
-  getting help in [#git
-  help](https://chat.zulip.org/#narrow/channel/44-git-help) if you run
-  into any troubles. Even Git experts should read the [Zulip-specific
-  Git tools
-  page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
-
-[graphical Git viewer]: https://zulip.readthedocs.io/en/latest/git/setup.html#get-a-graphical-client
-[git-log-secret]: https://github.com/zulip/zulip-mobile/blob/main/docs/howto/git.md#git-log-secret
-
-### Where to look for an issue
-
-Now you're ready to pick your first issue! Zulip has several repositories you
-can check out, depending on your interests. There are hundreds of open issues in
-the [main Zulip server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-alone.
-
-You can look through issues tagged with the "help wanted" label, which is used
-to indicate the issues that are ready for contributions. Some repositories also
-use the "good first issue" label to tag issues that are especially approachable
-for new contributors.
-
-- [Server and web app](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- Mobile apps: no "help wanted" label, but see the
-  [project board](https://github.com/orgs/zulip/projects/5/views/4)
-  for the upcoming Flutter-based app. Look for issues up through the
-  "Launch" milestone, and that aren't already assigned.
-- [Desktop app](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-- [Terminal app](https://github.com/zulip/zulip-terminal/issues?q=is%3Aopen+is%3Aissue+label%3A"help+wanted")
-- [Python API bindings and bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-
-### Picking an issue to work on
-
-There's a lot to learn while making your first pull request, so start small!
-Many first contributions have fewer than 10 lines of changes (not counting
-changes to tests).
-
-We recommend the following process for finding an issue to work on:
-
-1. Read the description of an issue tagged with the "help wanted" label and make
-   sure you understand it.
-2. If it seems promising, poke around the product
-   (on [chat.zulip.org](https://chat.zulip.org) or in the development
-   environment) until you know how the piece being
-   described fits into the bigger picture. If after some exploration the
-   description seems confusing or ambiguous, post a question on the GitHub
-   issue, as others may benefit from the clarification as well.
-3. When you find an issue you like, try to get started working on it. See if you
-   can find the part of the code you'll need to modify (`git grep` is your
-   friend!) and get some idea of how you'll approach the problem.
-4. If you feel lost, that's OK! Go through these steps again with another issue.
-   There's plenty to work on, and the exploration you do will help you learn
-   more about the project.
-
-Note that you are _not_ claiming an issue while you are iterating through steps
-1-4. _Before you claim an issue_, you should be confident that you will be able to
-tackle it effectively.
-
-Additional tips for the [main server and web app
-repository](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22):
-
-- We especially recommend browsing recently opened issues, as there are more
-  likely to be easy ones for you to find.
-- Take a look at issues with the ["good first issue"
-  label](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22),
-  as they are especially accessible to new contributors. However, you will
-  likely find issues without this label that are accessible as well.
-- All issues are partitioned into areas like
-  admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
-  through our [list of labels](https://github.com/zulip/zulip/labels), and
-  click on some of the `area:` labels to see all the issues related to your
-  areas of interest.
-- Avoid issues with the "difficult" label unless you
-  understand why it is difficult and are highly confident you can resolve the
-  issue correctly and completely.
-
-### Claiming an issue
-
-#### In the main server/web app repository and Zulip Terminal repository
-
-The Zulip server/web app repository
-([`zulip/zulip`](https://github.com/zulip/zulip/)) and the Zulip Terminal
-repository ([`zulip/zulip-terminal`](https://github.com/zulip/zulip-terminal/))
-are set up with a GitHub workflow bot called
-[Zulipbot](https://github.com/zulip/zulipbot), which manages issues and pull
-requests in order to create a better workflow for Zulip contributors.
-
-To claim an issue in these repositories, simply post a comment that says
-`@zulipbot claim` to the issue thread. If the issue is tagged with a [help
-wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
-label, Zulipbot will immediately assign the issue to you.
-
-Note that new contributors can only claim one issue until their first pull request is
-merged. This is to encourage folks to finish ongoing work before starting
-something new. If you would like to pick up a new issue while waiting for review
-on an almost-ready pull request, you can post a comment to this effect on the
-issue you're interested in.
-
-#### In other Zulip repositories
-
-There is no bot for other Zulip repositories
-([`zulip/zulip-flutter`](https://github.com/zulip/zulip-flutter/), etc.). If
-you are interested in claiming an issue in one of these repositories, simply
-post a comment on the issue thread saying that you'd like to work on it. There
-is no need to @-mention the issue creator in your comment.
-
-Please follow the same guidelines as described above: find an issue labeled
-"help wanted", and only pick up one issue at a time to start with.
-
-### Working on an issue
-
-You're encouraged to ask questions on how to best implement or debug your
-changes -- the Zulip maintainers are excited to answer questions to help you
-stay unblocked and working efficiently. You can ask questions in the [Zulip
-development community](https://zulip.com/development-community/), or on the
-GitHub issue or pull request.
-
-To get early feedback on any UI changes, we encourage you to post screenshots of
-your work in the [#design
-stream](https://chat.zulip.org/#narrow/channel/101-design) in the [Zulip
-development community](https://zulip.com/development-community/)
-
-For more advice, see [What makes a great Zulip
-contributor?](#what-makes-a-great-zulip-contributor) below. It's OK if your
-first issue takes you a while; that's normal! You'll be able to work a lot
-faster as you build experience.
-
-### Submitting a pull request
-
-See the [guide on submitting a pull
-request](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html)
-for detailed instructions on how to present your proposed changes to Zulip.
-
-The [pull request review process
-guide](https://zulip.readthedocs.io/en/latest/contributing/review-process.html)
-explains the stages of review your PR will go through, and offers guidance on
-how to help the review process move forward.
-
-### Beyond the first issue
-
-To find a second issue to work on, we recommend looking through issues with the same
-`area:` label as the last issue you resolved. You'll be able to reuse the
-work you did learning how that part of the codebase works. Also, the path to
-becoming a core developer often involves taking ownership of one of these area
-labels.
-
-### Common questions
-
-- **What if somebody is already working on the issue I want do claim?** There
-  are lots of issue to work on! If somebody else is actively working on the
-  issue, you can find a different one, or help with
-  reviewing their work.
-- **What if somebody else claims an issue while I'm figuring out whether or not to
-  work on it?** No worries! You can contribute by providing feedback on
-  their pull request. If you've made good progress in understanding part of the
-  codebase, you can also find another "help wanted" issue in the same area to
-  work on.
-- **What if there is already a pull request for the issue I want to work on?**
-  Start by reviewing the existing work. If you agree with the approach, you can
-  use the existing pull request (PR) as a starting point for your contribution. If
-  you think a different approach is needed, you can post a new PR, with a comment that clearly
-  explains _why_ you decided to start from scratch.
-- **What if I ask if someone is still working on an issue, and they don't
-  respond?** If you don't get a reply within 2-3 days, go ahead and post a comment
-  that you are working on the issue, and submit a pull request. If the original
-  assignee ends up submitting a pull request first, no worries! You can help by
-  providing feedback on their work, or submit your own PR if you think a
-  different approach is needed (as described above).
-- **Can I come up with my own feature idea and work on it?** We welcome
-  suggestions of features or other improvements that you feel would be valuable. If you
-  have a new feature you'd like to add, you can start a conversation [in our
-  development community](https://zulip.com/development-community/#where-do-i-send-my-message)
-  explaining the feature idea and the problem that you're hoping to solve.
-- **I'm waiting for the next round of review on my PR. Can I pick up
-  another issue in the meantime?** Someone's first Zulip PR often
-  requires quite a bit of iteration, so please [make sure your pull
-  request is reviewable][reviewable-pull-requests] and go through at
-  least one round of feedback from others before picking up a second
-  issue. After that, sure! If
-  [Zulipbot](https://github.com/zulip/zulipbot) does not allow you to
-  claim an issue, you can post a comment describing the status of your
-  other work on the issue you're interested in, and asking for the
-  issue to be assigned to you. Note that addressing feedback on
-  in-progress PRs should always take priority over starting a new PR.
-- **I think my PR is done, but it hasn't been merged yet. What's going on?**
-  1. **Double-check that you have addressed all the feedback**, including any comments
-     on [Git commit
-     discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
-  2. If all the feedback has been addressed, did you [leave a
-     comment](https://zulip.readthedocs.io/en/latest/contributing/review-process.html#how-to-help-move-the-review-process-forward)
-     explaining that you have done so and **requesting another review**? If not,
-     it may not be clear to project maintainers or reviewers that your PR is
-     ready for another look.
-  3. There may be a pause between initial rounds of review for your PR and final
-     review by project maintainers. This is normal, and we encourage you to **work
-     on other issues** while you wait.
-  4. If you think the PR is ready and haven't seen any updates for a couple
-     of weeks, it can be helpful to **leave another comment**. Summarize the
-     overall state of the review process and your work, and indicate that you
-     are waiting for a review.
-  5. Finally, **Zulip project maintainers are people too**! They may be busy
-     with other work, and sometimes they might even take a vacation. ;) It can
-     occasionally take a few weeks for a PR in the final stages of the review
-     process to be merged.
-
-[reviewable-pull-requests]: https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html
-
-## What makes a great Zulip contributor?
-
-Zulip has a lot of experience working with new contributors. In our
-experience, these are the best predictors of success:
-
-- [Asking great questions][great-questions]. It's very hard to answer a general
-  question like, "How do I do this issue?" When asking for help, explain your
-  current understanding, including what you've done or tried so far and where
-  you got stuck. Post tracebacks or other error messages if appropriate. For
-  more advice, check out [our guide][great-questions]!
-- Learning and practicing
-  [Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/commit-discipline.html).
-- Submitting carefully tested code. See our [detailed guide on how to review
-  code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#how-to-review-code)
-  (yours or someone else's).
-- Posting
-  [screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
-  for frontend changes.
-- Working to [make your pull requests easy to
-  review](https://zulip.readthedocs.io/en/latest/contributing/reviewable-prs.html).
-- Clearly describing what you have implemented and why. For example, if your
-  implementation differs from the issue description in some way or is a partial
-  step towards the requirements described in the issue, be sure to call
-  out those differences.
-- Being responsive to feedback on pull requests. This means incorporating or
-  responding to all suggested changes, and leaving a note if you won't be
-  able to address things within a few days.
-- Being helpful and friendly on the [Zulip community
-  server](https://zulip.com/development-community/).
-
-[great-questions]: https://zulip.readthedocs.io/en/latest/contributing/asking-great-questions.html
-
-## User feedback
-
-Nearly every feature we develop starts with a user request. If you are part
-of a group that is either using or considering using Zulip, we would love to
-hear about your experience with the product. If you're not sure what to
-write, here are some questions we're always very curious to know the answer
-to:
-
-- Evaluation: What is the process by which your organization chose or will
-  choose a group chat product?
-- Pros and cons: What are the pros and cons of Zulip for your organization,
-  and the pros and cons of other products you are evaluating?
-- Features: What are the features that are most important for your
-  organization? In the best-case scenario, what would your chat solution do
-  for you?
-- Onboarding: If you remember it, what was your impression during your first
-  few minutes of using Zulip? What did you notice, and how did you feel? Was
-  there anything that stood out to you as confusing, or broken, or great?
-- Organization: What does your organization do? How big is the organization?
-  A link to your organization's website?
-
-You can contact us in the [#feedback stream of the Zulip development
-community](https://chat.zulip.org/#narrow/channel/137-feedback) or
-by emailing [support@zulip.com](mailto:support@zulip.com).
-
-## Outreach programs
-
-Zulip regularly participates in [Google Summer of Code
-(GSoC)](https://developers.google.com/open-source/gsoc/) and
-[Outreachy](https://www.outreachy.org/). We have been a GSoC mentoring
-organization since 2016, and we accept 15-20 GSoC participants each summer. In
-the past, we’ve also participated in [Google
-Code-In](https://developers.google.com/open-source/gci/), and hosted summer
-interns from Harvard, MIT, and Stanford.
-
-Check out our [outreach programs
-overview](https://zulip.readthedocs.io/en/latest/outreach/overview.html) to learn
-more about participating in an outreach program with Zulip. Most of our program
-participants end up sticking around the project long-term, and many have become
-core team members, maintaining important parts of the project. We hope you
-apply!
-
-## Stay connected
-
-Even if you are not logging into the development community on a regular basis,
-you can still stay connected with the project.
-
-- Follow us [on Twitter](https://twitter.com/zulip).
-- Subscribe to [our blog](https://blog.zulip.org/).
-- Join or follow the project [on LinkedIn](https://www.linkedin.com/company/zulip-project/).
-
-## Help others find Zulip
-
-Here are some ways you can help others find Zulip:
-
-- Star us on GitHub. There are four main repositories:
-  [server/web](https://github.com/zulip/zulip),
-  [Flutter mobile](https://github.com/zulip/zulip-flutter),
-  [desktop](https://github.com/zulip/zulip-desktop), and
-  [Python API](https://github.com/zulip/python-zulip-api).
-
-- "Like" and retweet [our tweets](https://twitter.com/zulip).
-
-- Upvote and post feedback on Zulip on comparison websites. A couple specific
-  ones to highlight:
-
-  - [AlternativeTo](https://alternativeto.net/software/zulip-chat-server/). You can also
-    [upvote Zulip](https://alternativeto.net/software/slack/) on their page
-    for Slack.
-  - [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
-    it, and upvote the reasons why people like Zulip that you find most
-    compelling.

Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:trusty
+
+EXPOSE 9991
+
+RUN apt-get update && apt-get install -y \
+  python-pbs \
+  wget
+
+RUN locale-gen en_US.UTF-8
+
+RUN useradd -d /home/zulip -m zulip && echo 'zulip ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER zulip
+
+RUN ln -nsf /srv/zulip ~/zulip
+
+RUN echo 'export LC_ALL="en_US.UTF-8" LANG="en_US.UTF-8" LANGUAGE="en_US.UTF-8"' >> ~zulip/.bashrc
+
+WORKDIR /srv/zulip

Dockerfile-postgresql

@@ -1,25 +0,0 @@
-# This is a multiarch Dockerfile.  See https://docs.docker.com/desktop/multi-arch/
-#
-# To set up the first time:
-#     docker buildx create --name multiarch --use
-#
-# To build:
-#     docker buildx build --platform linux/amd64,linux/arm64 \
-#       -f ./Dockerfile-postgresql -t zulip/zulip-postgresql:14 --push .
-
-# Currently the PostgreSQL images do not support automatic upgrading of
-# the on-disk data in volumes. So the base image cannot currently be upgraded
-# without users needing a manual pgdump and restore.
-
-# https://hub.docker.com/r/groonga/pgroonga/tags
-ARG PGROONGA_VERSION=latest
-ARG POSTGRESQL_VERSION=14
-FROM groonga/pgroonga:$PGROONGA_VERSION-alpine-$POSTGRESQL_VERSION-slim
-
-# Install hunspell, Zulip stop words, and run Zulip database
-# init.
-RUN apk add -U --no-cache hunspell-en
-RUN ln -sf /usr/share/hunspell/en_US.dic /usr/local/share/postgresql/tsearch_data/en_us.dict && ln -sf /usr/share/hunspell/en_US.aff /usr/local/share/postgresql/tsearch_data/en_us.affix
-COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/local/share/postgresql/tsearch_data/zulip_english.stop
-COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
-COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql

NOTICE

@@ -1,18 +0,0 @@
-Copyright 2012–2015 Dropbox, Inc., 2015–2021 Kandra Labs, Inc., and contributors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this project except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-The software includes some works released by third parties under other
-free and open source licenses. Those works are redistributed under the
-license terms under which the works were received. For more details,
-see the ``docs/THIRDPARTY`` file included with this distribution.

README.md

@@ -1,82 +1,263 @@
+**[Zulip overview](#zulip-overview)** |
+**[Community](#community)** |
+**[Installing for dev](#installing-the-zulip-development-environment)** |
+**[Installing for production](#running-zulip-in-production)** |
+**[Ways to contribute](#ways-to-contribute)** |
+**[How to get involved](#how-to-get-involved-with-contributing-to-zulip)** |
+**[License](#license)**
+
 # Zulip overview
 
-[Zulip](https://zulip.com) is an open-source team collaboration tool with unique
+Zulip is a powerful, open source group chat application. Written in
-[topic-based threading][why-zulip] that combines the best of email and chat to
+Python and using the Django framework, Zulip supports both private
-make remote work productive and delightful. Fortune 500 companies, [leading open
+messaging and group chats via conversation streams.
-source projects][rust-case-study], and thousands of other organizations use
-Zulip every day. Zulip is the only [modern team chat app][features] that is
-designed for both live and asynchronous conversations.
 
-Zulip is built by a distributed community of developers from all around the
+Zulip also supports fast search, drag-and-drop file uploads, image
-world, with 74+ people who have each contributed 100+ commits. With
+previews, group private messages, audible notifications,
-over 1000 contributors merging over 500 commits a month, Zulip is the
+missed-message emails, desktop apps, and much more.
-largest and fastest growing open source team chat project.
 
-Come find us on the [development community chat](https://zulip.com/development-community/)!
+Further information on the Zulip project and its features can be found
+at https://www.zulip.org.
 
-[![GitHub Actions build status](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml/badge.svg)](https://github.com/zulip/zulip/actions/workflows/zulip-ci.yml?query=branch%3Amain)
+[![Build Status](https://travis-ci.org/zulip/zulip.svg?branch=master)](https://travis-ci.org/zulip/zulip) [![Coverage Status](https://coveralls.io/repos/github/zulip/zulip/badge.svg?branch=master)](https://coveralls.io/github/zulip/zulip?branch=master) [![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](http://zulip.readthedocs.io/en/latest/) [![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![coverage status](https://img.shields.io/codecov/c/github/zulip/zulip/main.svg)](https://codecov.io/gh/zulip/zulip)
-[![Mypy coverage](https://img.shields.io/badge/mypy-100%25-green.svg)][mypy-coverage]
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
-[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![GitHub release](https://img.shields.io/github/release/zulip/zulip.svg)](https://github.com/zulip/zulip/releases/latest)
-[![docs](https://readthedocs.org/projects/zulip/badge/?version=latest)](https://zulip.readthedocs.io/en/latest/)
-[![Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://chat.zulip.org)
-[![Twitter](https://img.shields.io/badge/twitter-@zulip-blue.svg?style=flat)](https://twitter.com/zulip)
-[![GitHub Sponsors](https://img.shields.io/github/sponsors/zulip)](https://github.com/sponsors/zulip)
 
-[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
+## Community
-[why-zulip]: https://zulip.com/why-zulip/
-[rust-case-study]: https://zulip.com/case-studies/rust/
-[features]: https://zulip.com/features/
 
-## Getting started
+There are several places online where folks discuss Zulip.
 
-- **Contributing code**. Check out our [guide for new
+One of those places is our [public Zulip instance](https://chat.zulip.org/).
-  contributors](https://zulip.readthedocs.io/en/latest/contributing/contributing.html)
+You can go through the simple signup process at that link, and then you
-  to get started. We have invested in making Zulip’s code highly
+will soon be talking to core Zulip developers and other users.  To get
-  readable, thoughtfully tested, and easy to modify. Beyond that, we
+help in real time, you will have the best luck finding core developers
-  have written an extraordinary 150K words of documentation for Zulip
+roughly between 16:00 UTC and 23:59 UTC.  Most questions get a reply
-  contributors.
+within minutes to a few hours, depending on time of day.
 
-- **Contributing non-code**. [Report an
+For Google Summer of Code students and applicants, we have [a mailing
-  issue](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#reporting-issues),
+list](https://groups.google.com/forum/#!forum/zulip-gsoc) for help,
-  [translate](https://zulip.readthedocs.io/en/latest/translating/translating.html)
+questions, and announcements.
-  Zulip into your language, or [give us
-  feedback](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#user-feedback).
-  We'd love to hear from you, whether you've been using Zulip for years, or are just
-  trying it out for the first time.
 
-- **Checking Zulip out**. The best way to see Zulip in action is to drop by the
+We have
-  [Zulip community server](https://zulip.com/development-community/). We also
+[a public mailing list](https://groups.google.com/forum/#!forum/zulip-devel)
-  recommend reading about Zulip's [unique
+that is currently pretty low traffic because most discussions happen
-  approach](https://zulip.com/why-zulip/) to organizing conversations.
+in our public Zulip instance.  We use it to announce Zulip developer
+community gatherings and ask for feedback on major technical or design
+decisions.  It has several hundred subscribers, so you can use it to
+ask questions about features or possible bugs, but please don't use it
+ask for generic help getting started as a contributor (e.g. because
+you want to do Google Summer of Code).  The rest of this page covers
+how to get involved in the Zulip project in detail.
 
-- **Running a Zulip server**. Self-host Zulip directly on Ubuntu or Debian
+Zulip also has a [blog](https://blog.zulip.org/).
-  Linux, in [Docker](https://github.com/zulip/docker-zulip), or with prebuilt
-  images for [Digital Ocean](https://marketplace.digitalocean.com/apps/zulip) and
-  [Render](https://render.com/docs/deploy-zulip).
-  Learn more about [self-hosting Zulip](https://zulip.com/self-hosting/).
 
-- **Using Zulip without setting up a server**. Learn about [Zulip
+Last but not least, we use [GitHub](https://github.com/zulip/zulip) to
-  Cloud](https://zulip.com/plans/) hosting options. Zulip sponsors free [Zulip
+track Zulip-related issues (and store our code, of course).
-  Cloud Standard](https://zulip.com/plans/) for hundreds of worthy
+Anybody with a GitHub account should be able to create Issues there
-  organizations, including [fellow open-source
+pertaining to bugs or enhancement requests.  We also use Pull
-  projects](https://zulip.com/for/open-source/).
+Requests as our primary mechanism to receive code contributions.
 
-- **Participating in [outreach
+The Zulip community has a [Code of Conduct][code-of-conduct].
-  programs](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#outreach-programs)**
-  like [Google Summer of Code](https://developers.google.com/open-source/gsoc/)
-  and [Outreachy](https://www.outreachy.org/).
 
-- **Supporting Zulip**. Advocate for your organization to use Zulip, become a
+## Installing the Zulip Development environment
-  [sponsor](https://github.com/sponsors/zulip), write a review in the mobile app
-  stores, or [help others find
-  Zulip](https://zulip.readthedocs.io/en/latest/contributing/contributing.html#help-others-find-zulip).
 
-You may also be interested in reading our [blog](https://blog.zulip.org/), and
+The Zulip development environment is the recommended option for folks
-following us on [Twitter](https://twitter.com/zulip) and
+interested in trying out Zulip.  This is documented in [the developer
-[LinkedIn](https://www.linkedin.com/company/zulip-project/).
+installation guide][dev-install].
 
-Zulip is distributed under the
+## Running Zulip in production
-[Apache 2.0](https://github.com/zulip/zulip/blob/main/LICENSE) license.
+
+Zulip in production supports Ubuntu 14.04 Trusty and Ubuntu 16.04
+Xenial. Work is ongoing on adding support for additional
+platforms. The installation process is documented at
+https://zulip.org/server.html and in more detail in [the
+documentation](https://zulip.readthedocs.io/en/latest/prod-install.html).
+
+## Ways to contribute
+
+Zulip welcomes all forms of contributions!  The page documents the
+Zulip development process.
+
+* **Pull requests**. Before a pull request can be merged, you need to
+sign the [Dropbox Contributor License Agreement][cla].  Also,
+please skim our [commit message style guidelines][doc-commit-style].
+
+* **Testing**. The Zulip automated tests all run automatically when
+you submit a pull request, but you can also run them all in your
+development environment following the instructions in the [testing
+docs][doc-test]. You can also try out [our new desktop
+client][electron], which is in alpha; we'd appreciate testing and
+[feedback](https://github.com/zulip/zulip-electron/issues/new).
+
+* **Developer Documentation**.  Zulip has a growing collection of
+developer documentation on [Read The Docs][doc].  Recommended reading
+for new contributors includes the [directory structure][doc-dirstruct]
+and [new feature tutorial][doc-newfeat]. You can also improve
+[Zulip.org][z-org].
+
+* **Mailing lists and bug tracker**. Zulip has a [development
+discussion mailing list](#community) and uses [GitHub issues
+][gh-issues].  There are also lists for the [Android][email-android]
+and [iOS][email-ios] apps.  Feel free to send any questions or
+suggestions of areas where you'd love to see more documentation to the
+relevant list!  Please report any security issues you discover to
+zulip-security@googlegroups.com.
+
+* **App codebases**. This repository is for the Zulip server and web
+app (including most integrations); the [desktop][], [Android][], and
+[iOS][] apps, are separate repositories, as are our [experimental
+React Native iOS app][ios-exp] and [alpha Electron desktop
+app][electron].
+
+* **Glue code**. We maintain a [Hubot adapter][hubot-adapter] and several
+integrations ([Phabricator][phab], [Jenkins][], [Puppet][], [Redmine][],
+and [Trello][]), plus [node.js API bindings][node], an [isomorphic
+ JavaScript library][zulip-js], and a [full-text search PostgreSQL
+ extension][tsearch], as separate repos.
+
+* **Translations**.  Zulip is in the process of being translated into
+10+ languages, and we love contributions to our translations.  See our
+[translating documentation][transifex] if you're interested in
+contributing!
+
+[cla]: https://opensource.dropbox.com/cla/
+[code-of-conduct]: https://zulip.readthedocs.io/en/latest/code-of-conduct.html
+[dev-install]: https://zulip.readthedocs.io/en/latest/dev-overview.html
+[doc]: https://zulip.readthedocs.io/
+[doc-commit-style]: http://zulip.readthedocs.io/en/latest/version-control.html#commit-messages
+[doc-dirstruct]: http://zulip.readthedocs.io/en/latest/directory-structure.html
+[doc-newfeat]: http://zulip.readthedocs.io/en/latest/new-feature-tutorial.html
+[doc-test]: http://zulip.readthedocs.io/en/latest/testing.html
+[electron]: https://github.com/zulip/zulip-electron
+[gh-issues]: https://github.com/zulip/zulip/issues
+[desktop]: https://github.com/zulip/zulip-desktop
+[android]: https://github.com/zulip/zulip-android
+[ios]: https://github.com/zulip/zulip-ios
+[ios-exp]: https://github.com/zulip/zulip-mobile
+[email-android]: https://groups.google.com/forum/#!forum/zulip-android
+[email-ios]: https://groups.google.com/forum/#!forum/zulip-ios
+[hubot-adapter]: https://github.com/zulip/hubot-zulip
+[jenkins]: https://github.com/zulip/zulip-jenkins-plugin
+[node]: https://github.com/zulip/zulip-node
+[zulip-js]: https://github.com/zulip/zulip-js
+[phab]: https://github.com/zulip/phabricator-to-zulip
+[puppet]: https://github.com/matthewbarr/puppet-zulip
+[redmine]: https://github.com/zulip/zulip-redmine-plugin
+[trello]: https://github.com/zulip/trello-to-zulip
+[tsearch]: https://github.com/zulip/tsearch_extras
+[transifex]: https://zulip.readthedocs.io/en/latest/translating.html#testing-translations
+[z-org]: https://github.com/zulip/zulip.github.io
+
+## Google Summer of Code
+
+We participated in
+[GSoC](https://developers.google.com/open-source/gsoc/) last year and
+hope to do so again in 2017.  For guidance, please read
+[our GSoC instructions and ideas page](https://github.com/zulip/zulip.github.io/blob/master/gsoc-ideas.md)
+and feel free to email
+[our GSoC mailing list](https://groups.google.com/forum/#!forum/zulip-gsoc).
+
+## How to get involved with contributing to Zulip
+
+First, subscribe to the Zulip [development discussion mailing
+list](#community).
+
+The Zulip project uses a system of labels in our [issue
+tracker][gh-issues] to make it easy to find a project if you don't
+have your own project idea in mind or want to get some experience with
+working on Zulip before embarking on a larger project you have in
+mind:
+
+* [Integrations](https://github.com/zulip/zulip/labels/area%3A%20integrations).
+  Integrate Zulip with another piece of software and contribute it
+  back to the community!  Writing an integration can be a great first
+  contribution.  There's detailed documentation on how to write
+  integrations in [the Zulip integration writing
+  guide](https://zulip.readthedocs.io/en/latest/integration-guide.html).
+
+* [Bite Size](https://github.com/zulip/zulip/labels/bite%20size):
+  Smaller projects that might be a great first contribution.
+
+* [Documentation](https://github.com/zulip/zulip/labels/area%3A%20documentation):
+  The Zulip project loves contributions of new documentation.
+
+* [Help Wanted](https://github.com/zulip/zulip/labels/help%20wanted):
+  A broader list of projects that nobody is currently working on.
+
+* [Platform support](https://github.com/zulip/zulip/labels/Platform%20support):
+  These are open issues about making it possible to install Zulip on a
+  wider range of platforms.
+
+* [Bugs](https://github.com/zulip/zulip/labels/bug): Open bugs.
+
+* [Feature requests](https://github.com/zulip/zulip/labels/enhancement):
+  Browsing this list can be a great way to find feature ideas to
+  implement that other Zulip users are excited about.
+
+* [2016 roadmap milestone](http://zulip.readthedocs.io/en/latest/roadmap.html):
+  The projects that are
+  [priorities for the Zulip project](https://zulip.readthedocs.io/en/latest/roadmap.html).
+  These are great projects if you're looking to make an impact.
+
+Another way to find issues in Zulip is to take advantage of our
+"area:<foo>" convention in separating out issues.  We partition all of
+our issues into areas like admin, compose, emoji, hotkeys, i18n,
+onboarding, search, etc.  You can see this here:
+
+[https://github.com/zulip/zulip/labels]
+
+Click on any of the "area:" labels and you will see all the tickets
+related to your area of interest.
+
+If you're excited about helping with an open issue, just post on the
+conversation thread that you're working on it.  You're encouraged to
+ask questions on how to best implement or debug your changes -- the
+Zulip maintainers are excited to answer questions to help you stay
+unblocked and working efficiently.
+
+We also welcome suggestions of features that you feel would be
+valuable or changes that you feel would make Zulip a better open
+source project, and are happy to support you in adding new features or
+other user experience improvements to Zulip.
+
+If you have a new feature you'd like to add, we recommend you start by
+opening a GitHub issue about the feature idea explaining the problem
+that you're hoping to solve and that you're excited to work on it.  A
+Zulip maintainer will usually reply within a day with feedback on the
+idea, notes on any important issues or concerns, and and often tips on
+how to implement or test it.  Please feel free to ping the thread if
+you don't hear a response from the maintainers -- we try to be very
+responsive so this usually means we missed your message.
+
+For significant changes to the visual design, user experience, data
+model, or architecture, we highly recommend posting a mockup,
+screenshot, or description of what you have in mind to zulip-devel@ to
+get broad feedback before you spend too much time on implementation
+details.
+
+Finally, before implementing a larger feature, we highly recommend
+looking at the new feature tutorial and coding style guidelines on
+ReadTheDocs.
+
+Feedback on how to make this development process more efficient, fun,
+and friendly to new contributors is very welcome!  Just send an email
+to the Zulip Developers list with your thoughts.
+
+## License
+
+Copyright 2011-2016 Dropbox, Inc. and contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The software includes some works released by third parties under other
+free and open source licenses. Those works are redistributed under the
+license terms under which the works were received. For more details,
+see the ``docs/THIRDPARTY`` file included with this distribution.

SECURITY.md

@@ -1,37 +0,0 @@
-# Security policy
-
-## Reporting a vulnerability
-
-We love responsible reports of (potential) security issues in Zulip,
-whether in the latest release or our development branch.
-
-Our security contact is security@zulip.com. Reporters should expect a
-response within 24 hours.
-
-Please include details on the issue and how you'd like to be credited
-in our release notes when we publish the fix.
-
-Our [security model][security-model] document may be a helpful
-resource.
-
-## Security announcements
-
-We send security announcements to our [announcement mailing
-list](https://groups.google.com/g/zulip-announce). If you are running
-Zulip in production, you should subscribe, by clicking "Join group" at
-the top of that page.
-
-## Supported versions
-
-Zulip provides security support for the latest major release, in the
-form of minor security/maintenance releases.
-
-We work hard to make [upgrades][upgrades] reliable, so that there's no
-reason to run older major releases.
-
-See also our documentation on the [Zulip release
-lifecycle][release-lifecycle].
-
-[security-model]: https://zulip.readthedocs.io/en/latest/production/security-model.html
-[upgrades]: https://zulip.readthedocs.io/en/stable/production/upgrade.html#upgrading-to-a-release
-[release-lifecycle]: https://zulip.readthedocs.io/en/latest/overview/release-lifecycle.html

Vagrantfile

@@ -1,108 +1,110 @@
 # -*- mode: ruby -*-
 
-Vagrant.require_version ">= 2.2.6"
+VAGRANTFILE_API_VERSION = "2"
+
+def command?(name)
+  `which #{name} > /dev/null 2>&1`
+  $?.success?
+end
+
+if Vagrant::VERSION == "1.8.7" then
+    path = `which curl`
+    if path.include?('/opt/vagrant/embedded/bin/curl') then
+        puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 1.8.6 "\
+             "or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
+             "issue before provisioning. See "\
+             "https://github.com/mitchellh/vagrant/issues/7997 "\
+             "for reference."
+        exit
+    end
+end
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  # For LXC. VirtualBox hosts use a different box, described below.
+  config.vm.box = "fgrehm/trusty64-lxc"
 
-Vagrant.configure("2") do |config|
   # The Zulip development environment runs on 9991 on the guest.
   host_port = 9991
-  http_proxy = https_proxy = no_proxy = nil
+  http_proxy = https_proxy = no_proxy = ""
   host_ip_addr = "127.0.0.1"
 
-  # System settings for the virtual machine.
-  vm_num_cpus = "2"
-  vm_memory = "2048"
-
-  ubuntu_mirror = ""
-  vboxadd_version = nil
-
-  config.vm.box = "bento/ubuntu-22.04"
-
   config.vm.synced_folder ".", "/vagrant", disabled: true
-  config.vm.synced_folder ".", "/srv/zulip", docker_consistency: "z"
+  config.vm.synced_folder ".", "/srv/zulip"
 
-  vagrant_config_file = ENV["HOME"] + "/.zulip-vagrant-config"
+  vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
   if File.file?(vagrant_config_file)
     IO.foreach(vagrant_config_file) do |line|
       line.chomp!
       key, value = line.split(nil, 2)
       case key
-      when /^([#;]|$)/ # ignore comments
+      when /^([#;]|$)/; # ignore comments
       when "HTTP_PROXY"; http_proxy = value
       when "HTTPS_PROXY"; https_proxy = value
       when "NO_PROXY"; no_proxy = value
       when "HOST_PORT"; host_port = value.to_i
       when "HOST_IP_ADDR"; host_ip_addr = value
-      when "GUEST_CPUS"; vm_num_cpus = value
-      when "GUEST_MEMORY_MB"; vm_memory = value
-      when "UBUNTU_MIRROR"; ubuntu_mirror = value
-      when "VBOXADD_VERSION"; vboxadd_version = value
       end
     end
   end
 
-  if Vagrant.has_plugin?("vagrant-proxyconf")
-    if !http_proxy.nil?
-      config.proxy.http = http_proxy
-    end
-    if !https_proxy.nil?
-      config.proxy.https = https_proxy
-    end
-    if !no_proxy.nil?
-      config.proxy.no_proxy = no_proxy
-    end
-  elsif !http_proxy.nil? or !https_proxy.nil?
-    # This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
-    # We haven't figured out a workaround.
-    puts "You have specified value for proxy in ~/.zulip-vagrant-config file but did not " \
-         "install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install " \
-         "vagrant-proxyconf` in a terminal.  This error will appear twice."
-    exit
-  end
-
   config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
-  config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
+
-  # Specify Docker provider before VirtualBox provider so it's preferred.
+  if Vagrant.has_plugin?("vagrant-proxyconf")
-  config.vm.provider "docker" do |d, override|
+    if http_proxy != ""
-    override.vm.box = nil
+      config.proxy.http = http_proxy
-    d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
+    end
-    d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
+    if https_proxy != ""
-    if !ubuntu_mirror.empty?
+      config.proxy.https = https_proxy
-      d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
+    end
+    if https_proxy != ""
+      config.proxy.no_proxy = no_proxy
+    end
+  end
+
+  # Specify LXC provider before VirtualBox provider so it's preferred.
+  config.vm.provider "lxc" do |lxc|
+    if command? "lxc-ls"
+      LXC_VERSION = `lxc-ls --version`.strip unless defined? LXC_VERSION
+      if LXC_VERSION >= "1.1.0"
+        # Allow start without AppArmor, otherwise Box will not Start on Ubuntu 14.10
+        # see https://github.com/fgrehm/vagrant-lxc/issues/333
+        lxc.customize 'aa_allow_incomplete', 1
+      end
+      if LXC_VERSION >= "2.0.0"
+        lxc.backingstore = 'dir'
+      end
     end
-    d.has_ssh = true
-    d.create_args = ["--ulimit", "nofile=1024:65536"]
   end
 
   config.vm.provider "virtualbox" do |vb, override|
+    override.vm.box = "ubuntu/trusty64"
     # It's possible we can get away with just 1.5GB; more testing needed
-    vb.memory = vm_memory
+    vb.memory = 2048
-    vb.cpus = vm_num_cpus
+    vb.cpus = 2
-
-    if !vboxadd_version.nil?
-      override.vbguest.installer = Class.new(VagrantVbguest::Installers::Ubuntu) do
-        define_method(:host_version) do |reload = false|
-          VagrantVbguest::Version(vboxadd_version)
-        end
-      end
-      override.vbguest.allow_downgrade = true
-      override.vbguest.iso_path = "https://download.virtualbox.org/virtualbox/#{vboxadd_version}/VBoxGuestAdditions_#{vboxadd_version}.iso"
-    end
   end
 
-  config.vm.provider "hyperv" do |h, override|
+  config.vm.provider "vmware_fusion" do |vb, override|
-    h.memory = vm_memory
+    override.vm.box = "puphpet/ubuntu1404-x64"
-    h.maxmemory = vm_memory
+    vb.vmx["memsize"] = "2048"
-    h.cpus = vm_num_cpus
+    vb.vmx["numvcpus"] = "2"
   end
 
-  config.vm.provider "parallels" do |prl, override|
+$provision_script = <<SCRIPT
-    prl.memory = vm_memory
+set -x
-    prl.cpus = vm_num_cpus
+set -e
-  end
+set -o pipefail
+# If the host is running SELinux remount the /sys/fs/selinux directory as read only,
+# needed for apt-get to work.
+if [ -d "/sys/fs/selinux" ]; then
+  sudo mount -o remount,ro /sys/fs/selinux
+fi
+ln -nsf /srv/zulip ~/zulip
+/srv/zulip/tools/provision
+SCRIPT
 
   config.vm.provision "shell",
     # We want provision to be run with the permissions of the vagrant user.
     privileged: false,
-    path: "tools/setup/vagrant-provision",
+    inline: $provision_script
-    env: { "UBUNTU_MIRROR" => ubuntu_mirror }
 end

analytics/lib/fixtures.py

@@ -1,21 +1,20 @@
-from math import sqrt
+from __future__ import division, absolute_import
-from random import Random
 
+from zerver.models import Realm, UserProfile, Stream, Message
+from analytics.models import InstallationCount, RealmCount, UserCount, StreamCount
 from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
 
+from datetime import datetime
+from math import sqrt
+from random import gauss, random, seed
 
-def generate_time_series_data(
+from six.moves import range, zip
-    days: int = 100,
+
-    business_hours_base: float = 10,
+def generate_time_series_data(days=100, business_hours_base=10, non_business_hours_base=10,
-    non_business_hours_base: float = 10,
+                              growth=1, autocorrelation=0, spikiness=1, holiday_rate=0,
-    growth: float = 1,
+                              frequency=CountStat.DAY, is_gauge=False, random_seed=26):
-    autocorrelation: float = 0,
+    # type: (int, float, float, float, float, float, float, str, bool, int) -> List[int]
-    spikiness: float = 1,
-    holiday_rate: float = 0,
-    frequency: str = CountStat.DAY,
-    partial_sum: bool = False,
-    random_seed: int = 26,
-) -> list[int]:
     """
     Generate semi-realistic looking time series data for testing analytics graphs.
 
@@ -32,46 +31,39 @@ def generate_time_series_data(
         the variance.
     holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
     frequency -- Should be CountStat.HOUR or CountStat.DAY.
-    partial_sum -- If True, return partial sum of the series.
+    is_gauge -- If True, return partial sum of the series.
     random_seed -- Seed for random number generator.
     """
-    rng = Random(random_seed)
-
     if frequency == CountStat.HOUR:
-        length = days * 24
+        length = days*24
         seasonality = [non_business_hours_base] * 24 * 7
         for day in range(5):
             for hour in range(8):
-                seasonality[24 * day + hour] = business_hours_base
+                seasonality[24*day + hour] = business_hours_base
-        holidays = []
+        holidays  = []
         for i in range(days):
-            holidays.extend([rng.random() < holiday_rate] * 24)
+            holidays.extend([random() < holiday_rate] * 24)
     elif frequency == CountStat.DAY:
         length = days
-        seasonality = [8 * business_hours_base + 16 * non_business_hours_base] * 5 + [
+        seasonality = [8*business_hours_base + 16*non_business_hours_base] * 5 + \
-            24 * non_business_hours_base
+                      [24*non_business_hours_base] * 2
-        ] * 2
+        holidays = [random() < holiday_rate for i in range(days)]
-        holidays = [rng.random() < holiday_rate for i in range(days)]
     else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
+        raise ValueError("Unknown frequency: %s" % (frequency,))
     if length < 2:
-        raise AssertionError(
+        raise ValueError("Must be generating at least 2 data points. "
-            f"Must be generating at least 2 data points. Currently generating {length}"
+                         "Currently generating %s" % (length,))
-        )
+    growth_base = growth ** (1. / (length-1))
-    growth_base = growth ** (1.0 / (length - 1))
     values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
 
-    noise_scalars = [rng.gauss(0, 1)]
+    seed(random_seed)
+    noise_scalars = [gauss(0, 1)]
     for i in range(1, length):
-        noise_scalars.append(
+        noise_scalars.append(noise_scalars[-1]*autocorrelation + gauss(0, 1)*(1-autocorrelation))
-            noise_scalars[-1] * autocorrelation + rng.gauss(0, 1) * (1 - autocorrelation)
-        )
 
-    values = [
+    values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
-        0 if holiday else int(v + sqrt(v) * noise_scalar * spikiness)
+              for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
-        for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays, strict=False)
+    if is_gauge:
-    ]
-    if partial_sum:
         for i in range(1, length):
-            values[i] = values[i - 1] + values[i]
+            values[i] = values[i-1] + values[i]
     return [max(v, 0) for v in values]

analytics/lib/time_utils.py

@@ -1,18 +1,15 @@
-from datetime import datetime, timedelta
+from zerver.lib.timestamp import floor_to_hour, floor_to_day, timestamp_to_datetime
-
 from analytics.lib.counts import CountStat
-from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
 
+from datetime import datetime, timedelta
+from typing import List, Optional
 
 # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
 # If min_length is greater than 0, pads the list to the left.
 # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
 # and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
-def time_range(
+def time_range(start, end, frequency, min_length):
-    start: datetime, end: datetime, frequency: str, min_length: int | None
+    # type: (datetime, datetime, str, Optional[int]) -> List[datetime]
-) -> list[datetime]:
-    verify_UTC(start)
-    verify_UTC(end)
     if frequency == CountStat.HOUR:
         end = floor_to_hour(end)
         step = timedelta(hours=1)
@@ -20,14 +17,13 @@ def time_range(
         end = floor_to_day(end)
         step = timedelta(days=1)
     else:
-        raise AssertionError(f"Unknown frequency: {frequency}")
+        raise ValueError("Unknown frequency: %s" % (frequency,))
 
     times = []
     if min_length is not None:
-        start = min(start, end - (min_length - 1) * step)
+        start = min(start, end - (min_length-1)*step)
     current = end
     while current >= start:
         times.append(current)
         current -= step
-    times.reverse()
+    return list(reversed(times))
-    return times

analytics/management/commands/active_user_stats.py

@@ -0,0 +1,60 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from typing import Any
+
+from zerver.models import UserPresence, UserActivity
+from zerver.lib.utils import statsd, statsd_key
+
+from datetime import datetime, timedelta
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = """Sends active user statistics to statsd.
+
+    Run as a cron job that runs every 10 minutes."""
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        # Get list of all active users in the last 1 week
+        cutoff = datetime.now() - timedelta(minutes=30, hours=168)
+
+        users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
+
+        # Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
+        hour_buckets = [0.16, 2, 12, 24, 48, 168]
+        user_info = defaultdict(dict) # type: Dict[str, Dict[float, List[str]]]
+
+        for last_presence in users:
+            if last_presence.status == UserPresence.IDLE:
+                known_active = last_presence.timestamp - timedelta(minutes=30)
+            else:
+                known_active = last_presence.timestamp
+
+            for bucket in hour_buckets:
+                if bucket not in user_info[last_presence.user_profile.realm.string_id]:
+                    user_info[last_presence.user_profile.realm.string_id][bucket] = []
+                if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
+                    user_info[last_presence.user_profile.realm.string_id][bucket].append(last_presence.user_profile.email)
+
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
+
+        # Also do stats for how many users have been reading the app.
+        users_reading = UserActivity.objects.select_related().filter(query="/json/messages/flags")
+        user_info = defaultdict(dict)
+        for activity in users_reading:
+            for bucket in hour_buckets:
+                if bucket not in user_info[activity.user_profile.realm.string_id]:
+                    user_info[activity.user_profile.realm.string_id][bucket] = []
+                if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
+                    user_info[activity.user_profile.realm.string_id][bucket].append(activity.user_profile.email)
+        for realm, buckets in user_info.items():
+            print("Realm %s" % (realm,))
+            for hr, users in sorted(buckets.items()):
+                print("\tUsers reading for %s: %s" % (hr, len(users)))
+                statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))

analytics/management/commands/active_user_stats_by_day.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import datetime
+import pytz
+
+from optparse import make_option
+from typing import Any
+from django.core.management.base import BaseCommand, CommandParser
+from zerver.lib.statistics import activity_averages_during_day
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity for a given day."
+
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
+        parser.add_argument('--date', default=None, action='store',
+                            help="Day to query in format 2013-12-05.  Default is yesterday")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options["date"] is None:
+            date = datetime.datetime.now() - datetime.timedelta(days=1)
+        else:
+            date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
+        print("Activity data for", date)
+        print(activity_averages_during_day(date))
+        print("Please note that the total registered user count is a total for today")

analytics/management/commands/analyze_mit.py

@@ -0,0 +1,87 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from optparse import make_option
+from django.core.management.base import BaseCommand, CommandParser
+from zerver.models import Recipient, Message
+from zerver.lib.timestamp import timestamp_to_datetime
+import datetime
+import time
+import logging
+
+def compute_stats(log_level):
+    # type: (int) -> None
+    logger = logging.getLogger()
+    logger.setLevel(log_level)
+
+    one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
+    mit_query = Message.objects.filter(sender__realm__string_id="mit",
+                                       recipient__type=Recipient.STREAM,
+                                       pub_date__gt=one_week_ago)
+    for bot_sender_start in ["imap.", "rcmd.", "sys."]:
+        mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
+    # Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
+    mit_query = mit_query.exclude(sender__email__contains=("/"))
+    mit_query = mit_query.exclude(sender__email__contains=("aim.com"))
+    mit_query = mit_query.exclude(
+        sender__email__in=["rss@mit.edu", "bash@mit.edu", "apache@mit.edu",
+                           "bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
+                           "root@mit.edu", "nagios@mit.edu",
+                           "www-data|local-realm@mit.edu"])
+    user_counts = {} # type: Dict[str, Dict[str, int]]
+    for m in mit_query.select_related("sending_client", "sender"):
+        email = m.sender.email
+        user_counts.setdefault(email, {})
+        user_counts[email].setdefault(m.sending_client.name, 0)
+        user_counts[email][m.sending_client.name] += 1
+
+    total_counts = {} # type: Dict[str, int]
+    total_user_counts = {} # type: Dict[str, int]
+    for email, counts in user_counts.items():
+        total_user_counts.setdefault(email, 0)
+        for client_name, count in counts.items():
+            total_counts.setdefault(client_name, 0)
+            total_counts[client_name] += count
+            total_user_counts[email] += count
+
+    logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
+    top_percents = {} # type: Dict[int, float]
+    for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
+        top_percents[size] = 0.0
+    for i, email in enumerate(sorted(total_user_counts.keys(),
+                                     key=lambda x: -total_user_counts[x])):
+        percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
+                              total_user_counts[email], 1)
+        for size in top_percents.keys():
+            top_percents.setdefault(size, 0)
+            if i < size:
+                top_percents[size] += (percent_zulip * 1.0 / size)
+
+        logging.debug("%40s | %10s | %s%%" % (email, total_user_counts[email],
+                                              percent_zulip))
+
+    logging.info("")
+    for size in sorted(top_percents.keys()):
+        logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
+
+    grand_total = sum(total_counts.values())
+    print(grand_total)
+    logging.info("%15s | %s" % ("Client", "Percentage"))
+    for client in total_counts.keys():
+        logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
+
+class Command(BaseCommand):
+    help = "Compute statistics on MIT Zephyr usage."
+
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
+        parser.add_argument('--verbose', default=False, action='store_true')
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        level = logging.INFO
+        if options["verbose"]:
+            level = logging.DEBUG
+        compute_stats(level)

analytics/management/commands/analyze_user_activity.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any, Dict
+
+from zerver.lib.statistics import seconds_usage_between
+
+from optparse import make_option
+from django.core.management.base import BaseCommand, CommandParser
+from zerver.models import UserProfile
+import datetime
+from django.utils.timezone import utc
+
+def analyze_activity(options):
+    # type: (Dict[str, Any]) -> None
+    day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
+    day_end = day_start + datetime.timedelta(days=options["duration"])
+
+    user_profile_query = UserProfile.objects.all()
+    if options["realm"]:
+        user_profile_query = user_profile_query.filter(realm__string_id=options["realm"])
+
+    print("Per-user online duration:\n")
+    total_duration = datetime.timedelta(0)
+    for user_profile in user_profile_query:
+        duration = seconds_usage_between(user_profile, day_start, day_end)
+
+        if duration == datetime.timedelta(0):
+            continue
+
+        total_duration += duration
+        print("%-*s%s" % (37, user_profile.email, duration,))
+
+    print("\nTotal Duration:                      %s" % (total_duration,))
+    print("\nTotal Duration in minutes:           %s" % (total_duration.total_seconds() / 60.,))
+    print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
+
+class Command(BaseCommand):
+    help = """Report analytics of user activity on a per-user and realm basis.
+
+This command aggregates user activity data that is collected by each user using Zulip. It attempts
+to approximate how much each user has been using Zulip per day, measured by recording each 15 minute
+period where some activity has occurred (mouse move or keyboard activity).
+
+It will correctly not count server-initiated reloads in the activity statistics.
+
+The duration flag can be used to control how many days to show usage duration for
+
+Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--duration=1]
+
+By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
+is shown for all realms"""
+
+    def add_arguments(self, parser):
+        # type: (CommandParser) -> None
+        parser.add_argument('--realm', action='store')
+        parser.add_argument('--date', action='store', default="2013-09-06")
+        parser.add_argument('--duration', action='store', default=1, type=int,
+                            help="How many days to show usage information for")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        analyze_activity(options)

analytics/management/commands/check_analytics_state.py

@@ -1,92 +0,0 @@
-from dataclasses import dataclass
-from datetime import timedelta
-from typing import Any, Literal
-
-from django.utils.timezone import now as timezone_now
-from typing_extensions import override
-
-from analytics.lib.counts import ALL_COUNT_STATS, CountStat
-from analytics.models import installation_epoch
-from scripts.lib.zulip_tools import atomic_nagios_write
-from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.timestamp import TimeZoneNotUTCError, floor_to_day, floor_to_hour, verify_UTC
-from zerver.models import Realm
-
-states = {
-    0: "OK",
-    1: "WARNING",
-    2: "CRITICAL",
-    3: "UNKNOWN",
-}
-
-
-@dataclass
-class NagiosResult:
-    status: Literal["ok", "warning", "critical", "unknown"]
-    message: str
-
-
-class Command(ZulipBaseCommand):
-    help = """Checks FillState table.
-
-    Run as a cron job that runs every hour."""
-
-    @override
-    def handle(self, *args: Any, **options: Any) -> None:
-        fill_state = self.get_fill_state()
-        atomic_nagios_write("check-analytics-state", fill_state.status, fill_state.message)
-
-    def get_fill_state(self) -> NagiosResult:
-        if not Realm.objects.exists():
-            return NagiosResult(status="ok", message="No realms exist, so not checking FillState.")
-
-        warning_unfilled_properties = []
-        critical_unfilled_properties = []
-        for property, stat in ALL_COUNT_STATS.items():
-            last_fill = stat.last_successful_fill()
-            if last_fill is None:
-                last_fill = installation_epoch()
-            try:
-                verify_UTC(last_fill)
-            except TimeZoneNotUTCError:
-                return NagiosResult(
-                    status="critical", message=f"FillState not in UTC for {property}"
-                )
-
-            if stat.frequency == CountStat.DAY:
-                floor_function = floor_to_day
-                warning_threshold = timedelta(hours=26)
-                critical_threshold = timedelta(hours=50)
-            else:  # CountStat.HOUR
-                floor_function = floor_to_hour
-                warning_threshold = timedelta(minutes=90)
-                critical_threshold = timedelta(minutes=150)
-
-            if floor_function(last_fill) != last_fill:
-                return NagiosResult(
-                    status="critical",
-                    message=f"FillState not on {stat.frequency} boundary for {property}",
-                )
-
-            time_to_last_fill = timezone_now() - last_fill
-            if time_to_last_fill > critical_threshold:
-                critical_unfilled_properties.append(property)
-            elif time_to_last_fill > warning_threshold:
-                warning_unfilled_properties.append(property)
-
-        if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
-            return NagiosResult(status="ok", message="FillState looks fine.")
-        if len(critical_unfilled_properties) == 0:
-            return NagiosResult(
-                status="warning",
-                message="Missed filling {} once.".format(
-                    ", ".join(warning_unfilled_properties),
-                ),
-            )
-        return NagiosResult(
-            status="critical",
-            message="Missed filling {} once. Missed filling {} at least twice.".format(
-                ", ".join(warning_unfilled_properties),
-                ", ".join(critical_unfilled_properties),
-            ),
-        )

analytics/management/commands/clear_analytics_tables.py

@@ -1,25 +1,29 @@
-from argparse import ArgumentParser
+from __future__ import absolute_import
-from typing import Any
+from __future__ import print_function
 
-from django.core.management.base import CommandError
+import sys
-from typing_extensions import override
+
+from argparse import ArgumentParser
+from django.db import connection
+from django.core.management.base import BaseCommand
 
 from analytics.lib.counts import do_drop_all_analytics_tables
-from zerver.lib.management import ZulipBaseCommand
 
+from typing import Any
 
-class Command(ZulipBaseCommand):
+class Command(BaseCommand):
     help = """Clear analytics tables."""
 
-    @override
+    def add_arguments(self, parser):
-    def add_arguments(self, parser: ArgumentParser) -> None:
+        # type: (ArgumentParser) -> None
-        parser.add_argument("--force", action="store_true", help="Clear analytics tables.")
+        parser.add_argument('--force',
+                            action='store_true',
+                            help="Clear analytics tables.")
 
-    @override
+    def handle(self, *args, **options):
-    def handle(self, *args: Any, **options: Any) -> None:
+        # type: (*Any, **Any) -> None
-        if options["force"]:
+        if options['force']:
             do_drop_all_analytics_tables()
         else:
-            raise CommandError(
+            print("Would delete all data from analytics tables (!); use --force to do so.")
-                "Would delete all data from analytics tables (!); use --force to do so."
+            sys.exit(1)
-            )

analytics/management/commands/clear_single_stat.py

@@ -1,27 +0,0 @@
-from argparse import ArgumentParser
-from typing import Any
-
-from django.core.management.base import CommandError
-from typing_extensions import override
-
-from analytics.lib.counts import ALL_COUNT_STATS, do_drop_single_stat
-from zerver.lib.management import ZulipBaseCommand
-
-
-class Command(ZulipBaseCommand):
-    help = """Clear analytics tables."""
-
-    @override
-    def add_arguments(self, parser: ArgumentParser) -> None:
-        parser.add_argument("--force", action="store_true", help="Actually do it.")
-        parser.add_argument("--property", help="The property of the stat to be cleared.")
-
-    @override
-    def handle(self, *args: Any, **options: Any) -> None:
-        property = options["property"]
-        if property not in ALL_COUNT_STATS:
-            raise CommandError(f"Invalid property: {property}")
-        if not options["force"]:
-            raise CommandError("No action taken. Use --force.")
-
-        do_drop_single_stat(property)

analytics/management/commands/client_activity.py

@@ -0,0 +1,80 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
+from django.db.models import Count, QuerySet
+
+from zerver.models import UserActivity, UserProfile, Realm, \
+    get_realm, get_user_profile_by_email
+
+import datetime
+
+class Command(BaseCommand):
+    help = """Report rough client activity globally, for a realm, or for a user
+
+Usage examples:
+
+./manage.py client_activity
+./manage.py client_activity zulip
+./manage.py client_activity hamlet@zulip.com"""
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('arg', metavar='<arg>', type=str, nargs='?', default=None,
+                            help="realm or user to estimate client activity for")
+
+    def compute_activity(self, user_activity_objects):
+        # type: (QuerySet) -> None
+        # Report data from the past week.
+        #
+        # This is a rough report of client activity because we inconsistently
+        # register activity from various clients; think of it as telling you
+        # approximately how many people from a group have used a particular
+        # client recently. For example, this might be useful to get a sense of
+        # how popular different versions of a desktop client are.
+        #
+        # Importantly, this does NOT tell you anything about the relative
+        # volumes of requests from clients.
+        threshold = datetime.datetime.now() - datetime.timedelta(days=7)
+        client_counts = user_activity_objects.filter(
+            last_visit__gt=threshold).values("client__name").annotate(
+            count=Count('client__name'))
+
+        total = 0
+        counts = []
+        for client_type in client_counts:
+            count = client_type["count"]
+            client = client_type["client__name"]
+            total += count
+            counts.append((count, client))
+
+        counts.sort()
+
+        for count in counts:
+            print("%25s %15d" % (count[1], count[0]))
+        print("Total:", total)
+
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['arg'] is None:
+            # Report global activity.
+            self.compute_activity(UserActivity.objects.all())
+        else:
+            arg = options['arg']
+            try:
+                # Report activity for a user.
+                user_profile = get_user_profile_by_email(arg)
+                self.compute_activity(UserActivity.objects.filter(
+                    user_profile=user_profile))
+            except UserProfile.DoesNotExist:
+                try:
+                    # Report activity for a realm.
+                    realm = get_realm(arg)
+                    self.compute_activity(UserActivity.objects.filter(
+                        user_profile__realm=realm))
+                except Realm.DoesNotExist:
+                    print("Unknown user or realm %s" % (arg,))
+                    exit(1)

analytics/management/commands/populate_analytics_db.py

@@ -1,352 +1,123 @@
-from collections.abc import Mapping
+from __future__ import absolute_import, print_function
-from datetime import timedelta
-from typing import Any, TypeAlias
 
-from django.core.files.uploadedfile import UploadedFile
+from argparse import ArgumentParser
-from django.utils.timezone import now as timezone_now
-from typing_extensions import override
 
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from analytics.models import BaseCount, InstallationCount, RealmCount, \
+    UserCount, StreamCount
 from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables
 from analytics.lib.fixtures import generate_time_series_data
 from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    FillState,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-)
-from zerver.actions.create_realm import do_create_realm
-from zerver.actions.users import do_change_user_role
-from zerver.lib.create_user import create_user
-from zerver.lib.management import ZulipBaseCommand
-from zerver.lib.storage import static_path
-from zerver.lib.stream_color import STREAM_ASSIGNMENT_COLORS
 from zerver.lib.timestamp import floor_to_day
-from zerver.lib.upload import upload_message_attachment_from_request
+from zerver.models import Realm, UserProfile, Stream, Message, Client
-from zerver.models import (
-    Client,
-    NamedUserGroup,
-    Realm,
-    RealmAuditLog,
-    Recipient,
-    Stream,
-    Subscription,
-    UserProfile,
-)
-from zerver.models.groups import SystemGroups
 
+from datetime import datetime, timedelta
 
-class Command(ZulipBaseCommand):
+from six.moves import zip
+from typing import Any, List, Optional, Text, Type, Union
+
+class Command(BaseCommand):
     help = """Populates analytics tables with randomly generated data."""
 
     DAYS_OF_DATA = 100
     random_seed = 26
 
-    def generate_fixture_data(
+    def create_user(self, email, full_name, is_staff, date_joined, realm):
-        self,
+        # type: (Text, Text, Text, bool, datetime, Realm) -> UserProfile
-        stat: CountStat,
+        return UserProfile.objects.create(
-        business_hours_base: float,
+            email=email, full_name=full_name, is_staff=is_staff,
-        non_business_hours_base: float,
+            realm=realm, short_name=full_name, pointer=-1, last_pointer_updater='none',
-        growth: float,
+            api_key='42', date_joined=date_joined)
-        autocorrelation: float,
+
-        spikiness: float,
+    def generate_fixture_data(self, stat, business_hours_base, non_business_hours_base,
-        holiday_rate: float = 0,
+                              growth, autocorrelation, spikiness, holiday_rate=0):
-        partial_sum: bool = False,
+        # type: (CountStat, float, float, float, float, float, float) -> List[int]
-    ) -> list[int]:
         self.random_seed += 1
         return generate_time_series_data(
-            days=self.DAYS_OF_DATA,
+            days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
-            business_hours_base=business_hours_base,
+            non_business_hours_base=non_business_hours_base, growth=growth,
-            non_business_hours_base=non_business_hours_base,
+            autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
-            growth=growth,
+            frequency=stat.frequency, is_gauge=(stat.interval == CountStat.GAUGE),
-            autocorrelation=autocorrelation,
+            random_seed=self.random_seed)
-            spikiness=spikiness,
-            holiday_rate=holiday_rate,
-            frequency=stat.frequency,
-            partial_sum=partial_sum,
-            random_seed=self.random_seed,
-        )
 
-    @override
+    def handle(self, *args, **options):
-    def handle(self, *args: Any, **options: Any) -> None:
+        # type: (*Any, **Any) -> None
-        # TODO: This should arguably only delete the objects
-        # associated with the "analytics" realm.
         do_drop_all_analytics_tables()
+        # I believe this also deletes any objects with this realm as a foreign key
+        Realm.objects.filter(string_id='analytics').delete()
+        Client.objects.filter(name__endswith='_').delete()
 
-        # This also deletes any objects with this realm as a foreign key
+        installation_time = timezone.now() - timedelta(days=self.DAYS_OF_DATA)
-        Realm.objects.filter(string_id="analytics").delete()
+        last_end_time = floor_to_day(timezone.now())
+        realm = Realm.objects.create(
+            string_id='analytics', name='Analytics', domain='analytics.ds',
+            date_created=installation_time)
+        shylock = self.create_user('shylock@analytics.ds', 'Shylock', True, installation_time, realm)
 
-        # Because we just deleted a bunch of objects in the database
+        def insert_fixture_data(stat, fixture_data, table):
-        # directly (rather than deleting individual objects in Django,
+            # type: (CountStat, Dict[Optional[str], List[int]], Type[BaseCount]) -> None
-        # in which case our post_save hooks would have flushed the
+            end_times = time_range(last_end_time, last_end_time, stat.frequency,
-        # individual objects from memcached for us), we need to flush
+                                   len(list(fixture_data.values())[0]))
-        # memcached in order to ensure deleted objects aren't still
-        # present in the memcached cache.
-        from zerver.apps import flush_cache
-
-        flush_cache(None)
-
-        installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
-        last_end_time = floor_to_day(timezone_now())
-        realm = do_create_realm(
-            string_id="analytics", name="Analytics", date_created=installation_time
-        )
-
-        shylock = create_user(
-            "shylock@analytics.ds",
-            "Shylock",
-            realm,
-            full_name="Shylock",
-            role=UserProfile.ROLE_REALM_OWNER,
-            force_date_joined=installation_time,
-        )
-        do_change_user_role(shylock, UserProfile.ROLE_REALM_OWNER, acting_user=None)
-
-        # Create guest user for set_guest_users_statistic.
-        create_user(
-            "bassanio@analytics.ds",
-            "Bassanio",
-            realm,
-            full_name="Bassanio",
-            role=UserProfile.ROLE_GUEST,
-            force_date_joined=installation_time,
-        )
-
-        administrators_user_group = NamedUserGroup.objects.get(
-            name=SystemGroups.ADMINISTRATORS, realm=realm, is_system_group=True
-        )
-        stream = Stream.objects.create(
-            name="all",
-            realm=realm,
-            date_created=installation_time,
-            can_remove_subscribers_group=administrators_user_group,
-        )
-        recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
-        stream.recipient = recipient
-        stream.save(update_fields=["recipient"])
-
-        # Subscribe shylock to the stream to avoid invariant failures.
-        Subscription.objects.create(
-            recipient=recipient,
-            user_profile=shylock,
-            is_user_active=shylock.is_active,
-            color=STREAM_ASSIGNMENT_COLORS[0],
-        )
-        RealmAuditLog.objects.create(
-            realm=realm,
-            modified_user=shylock,
-            modified_stream=stream,
-            event_last_message_id=0,
-            event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
-            event_time=installation_time,
-        )
-
-        # Create an attachment in the database for set_storage_space_used_statistic.
-        IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
-        with open(IMAGE_FILE_PATH, "rb") as fp:
-            upload_message_attachment_from_request(UploadedFile(fp), shylock)
-
-        FixtureData: TypeAlias = Mapping[str | int | None, list[int]]
-
-        def insert_fixture_data(
-            stat: CountStat,
-            fixture_data: FixtureData,
-            table: type[BaseCount],
-        ) -> None:
-            end_times = time_range(
-                last_end_time, last_end_time, stat.frequency, len(next(iter(fixture_data.values())))
-            )
-            if table == InstallationCount:
-                id_args: dict[str, Any] = {}
             if table == RealmCount:
-                id_args = {"realm": realm}
+                id_args = {'realm': realm}
             if table == UserCount:
-                id_args = {"realm": realm, "user": shylock}
+                id_args = {'realm': realm, 'user': shylock}
-            if table == StreamCount:
-                id_args = {"stream": stream, "realm": realm}
-
             for subgroup, values in fixture_data.items():
-                table._default_manager.bulk_create(
+                table.objects.bulk_create([
-                    table(
+                    table(property=stat.property, subgroup=subgroup, end_time=end_time,
-                        property=stat.property,
+                          value=value, **id_args)
-                        subgroup=subgroup,
+                    for end_time, value in zip(end_times, values) if value != 0])
-                        end_time=end_time,
-                        value=value,
-                        **id_args,
-                    )
-                    for end_time, value in zip(end_times, values, strict=False)
-                    if value != 0
-                )
 
-        stat = COUNT_STATS["1day_actives::day"]
+        stat = COUNT_STATS['active_users:is_bot:day']
-        realm_data: FixtureData = {
-            None: self.generate_fixture_data(stat, 0.08, 0.02, 3, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data: FixtureData = {
-            None: self.generate_fixture_data(stat, 0.8, 0.2, 4, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["7day_actives::day"]
         realm_data = {
-            None: self.generate_fixture_data(stat, 0.2, 0.07, 3, 0.3, 6, partial_sum=True),
+            'false': self.generate_fixture_data(stat, .1, .03, 3, .5, 3),
-        }
+            'true': self.generate_fixture_data(stat, .01, 0, 1, 0, 1)
+        } # type: Dict[Optional[str], List[int]]
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 2, 0.7, 4, 0.3, 6, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
 
-        stat = COUNT_STATS["realm_active_humans::day"]
+        stat = COUNT_STATS['messages_sent:is_bot:hour']
-        realm_data = {
+        user_data = {'false': self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)}
-            None: self.generate_fixture_data(stat, 0.8, 0.08, 3, 0.5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            None: self.generate_fixture_data(stat, 1, 0.3, 4, 0.5, 3, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        realm_data = {
-            "false": self.generate_fixture_data(stat, 1, 0.2, 3.5, 0.8, 2, partial_sum=True),
-            "true": self.generate_fixture_data(stat, 0.3, 0.05, 3, 0.3, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "false": self.generate_fixture_data(stat, 3, 1, 4, 0.8, 2, partial_sum=True),
-            "true": self.generate_fixture_data(stat, 1, 0.4, 4, 0.8, 2, partial_sum=True),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        user_data: FixtureData = {
-            "false": self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8, holiday_rate=0.1),
-        }
         insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {
+        realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
-            "false": self.generate_fixture_data(stat, 35, 15, 6, 0.6, 4),
+                      'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
-            "true": self.generate_fixture_data(stat, 15, 15, 3, 0.4, 2),
-        }
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "false": self.generate_fixture_data(stat, 350, 150, 6, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 150, 150, 3, 0.4, 2),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
 
-        stat = COUNT_STATS["messages_sent:message_type:day"]
+        stat = COUNT_STATS['messages_sent:message_type:day']
         user_data = {
-            "public_stream": self.generate_fixture_data(stat, 1.5, 1, 3, 0.6, 8),
+            'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
-            "private_message": self.generate_fixture_data(stat, 0.5, 0.3, 1, 0.6, 8),
+            'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8)}
-            "huddle_message": self.generate_fixture_data(stat, 0.2, 0.2, 2, 0.6, 8),
-        }
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
-            "public_stream": self.generate_fixture_data(stat, 30, 8, 5, 0.6, 4),
+            'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
-            "private_stream": self.generate_fixture_data(stat, 7, 7, 5, 0.6, 4),
+            'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
-            "private_message": self.generate_fixture_data(stat, 13, 5, 5, 0.6, 4),
+            'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4)}
-            "huddle_message": self.generate_fixture_data(stat, 6, 3, 3, 0.6, 4),
-        }
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            "public_stream": self.generate_fixture_data(stat, 300, 80, 5, 0.6, 4),
-            "private_stream": self.generate_fixture_data(stat, 70, 70, 5, 0.6, 4),
-            "private_message": self.generate_fixture_data(stat, 130, 50, 5, 0.6, 4),
-            "huddle_message": self.generate_fixture_data(stat, 60, 30, 3, 0.6, 4),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
 
-        website, created = Client.objects.get_or_create(name="website")
+        website_ = Client.objects.create(name='website_')
-        old_desktop, created = Client.objects.get_or_create(name="desktop app Linux 0.3.7")
+        API_ = Client.objects.create(name='API_')
-        android, created = Client.objects.get_or_create(name="ZulipAndroid")
+        android_ = Client.objects.create(name='android_')
-        iOS, created = Client.objects.get_or_create(name="ZulipiOS")
+        iOS_ = Client.objects.create(name='iOS_')
-        react_native, created = Client.objects.get_or_create(name="ZulipMobile")
+        react_native_ = Client.objects.create(name='react_native_')
-        flutter, created = Client.objects.get_or_create(name="ZulipFlutter")
+        electron_ = Client.objects.create(name='electron_')
-        API, created = Client.objects.get_or_create(name="API: Python")
+        barnowl_ = Client.objects.create(name='barnowl_')
-        zephyr_mirror, created = Client.objects.get_or_create(name="zephyr_mirror")
+        plan9_ = Client.objects.create(name='plan9_')
-        unused, created = Client.objects.get_or_create(name="unused")
-        long_webhook, created = Client.objects.get_or_create(name="ZulipLooooooooooongNameWebhook")
 
-        stat = COUNT_STATS["messages_sent:client:day"]
+        stat = COUNT_STATS['messages_sent:client:day']
         user_data = {
-            website.id: self.generate_fixture_data(stat, 2, 1, 1.5, 0.6, 8),
+            website_.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 0, 0.3, 1.5, 0.6, 8),
+            barnowl_.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
-        }
         insert_fixture_data(stat, user_data, UserCount)
         realm_data = {
-            website.id: self.generate_fixture_data(stat, 30, 20, 5, 0.6, 3),
+            website_.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, 0.6, 3),
+            API_.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
-            android.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
+            android_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            iOS.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
+            iOS_.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            react_native_.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
+            electron_.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
-            API.id: self.generate_fixture_data(stat, 5, 5, 5, 0.6, 3),
+            barnowl_.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, 0.6, 3),
+            plan9_.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0, 0)}
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, 0.6, 3),
-        }
         insert_fixture_data(stat, realm_data, RealmCount)
-        installation_data = {
-            website.id: self.generate_fixture_data(stat, 300, 200, 5, 0.6, 3),
-            old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, 0.6, 3),
-            android.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            iOS.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-            flutter.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            react_native.id: self.generate_fixture_data(stat, 5, 5, 10, 0.6, 3),
-            API.id: self.generate_fixture_data(stat, 50, 50, 5, 0.6, 3),
-            zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, 0.6, 3),
-            unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
-            long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, 0.6, 3),
-        }
-        insert_fixture_data(stat, installation_data, InstallationCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
 
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
+        # TODO: messages_sent_to_stream:is_bot
-        realm_data = {
-            "false": self.generate_fixture_data(stat, 30, 5, 6, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 20, 2, 3, 0.2, 3),
-        }
-        insert_fixture_data(stat, realm_data, RealmCount)
-        stream_data: Mapping[int | str | None, list[int]] = {
-            "false": self.generate_fixture_data(stat, 10, 7, 5, 0.6, 4),
-            "true": self.generate_fixture_data(stat, 5, 3, 2, 0.4, 2),
-        }
-        insert_fixture_data(stat, stream_data, StreamCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )
-
-        stat = COUNT_STATS["messages_read::hour"]
-        user_data = {
-            None: self.generate_fixture_data(stat, 7, 3, 2, 0.6, 8, holiday_rate=0.1),
-        }
-        insert_fixture_data(stat, user_data, UserCount)
-        realm_data = {None: self.generate_fixture_data(stat, 50, 35, 6, 0.6, 4)}
-        insert_fixture_data(stat, realm_data, RealmCount)
-        FillState.objects.create(
-            property=stat.property, end_time=last_end_time, state=FillState.DONE
-        )

analytics/management/commands/realm_stats.py

@@ -0,0 +1,167 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+import datetime
+import pytz
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
+    Subscription, UserMessage, get_realm
+
+MOBILE_CLIENT_LIST = ["Android", "ios"]
+HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
+
+human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LIST)
+
+class Command(BaseCommand):
+    help = "Generate statistics on realm activity."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def active_users(self, realm):
+        # type: (Realm) -> List[UserProfile]
+        # Has been active (on the website, for now) in the last 7 days.
+        activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
+        return [activity.user_profile for activity in (
+            UserActivity.objects.filter(user_profile__realm=realm,
+                                        user_profile__is_active=True,
+                                        last_visit__gt=activity_cutoff,
+                                        query="/json/users/me/pointer",
+                                        client__name="website"))]
+
+    def messages_sent_by(self, user, days_ago):
+        # type: (UserProfile, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
+
+    def total_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
+
+    def human_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
+
+    def api_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
+
+    def stream_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
+                                     recipient__type=Recipient.STREAM).count()
+
+    def private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
+            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
+
+    def group_private_messages(self, realm, days_ago):
+        # type: (Realm, int) -> int
+        sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
+        return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
+            recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
+
+    def report_percentage(self, numerator, denominator, text):
+        # type: (float, float, str) -> None
+        if not denominator:
+            fraction = 0.0
+        else:
+            fraction = numerator / float(denominator)
+        print("%.2f%% of" % (fraction * 100,), text)
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(string_id) for string_id in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.string_id)
+
+            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
+            active_users = self.active_users(realm)
+            num_active = len(active_users)
+
+            print("%d active users (%d total)" % (num_active, len(user_profiles)))
+            streams = Stream.objects.filter(realm=realm).extra(
+                tables=['zerver_subscription', 'zerver_recipient'],
+                where=['zerver_subscription.recipient_id = zerver_recipient.id',
+                       'zerver_recipient.type = 2',
+                       'zerver_recipient.type_id = zerver_stream.id',
+                       'zerver_subscription.active = true']).annotate(count=Count("name"))
+            print("%d streams" % (streams.count(),))
+
+            for days_ago in (1, 7, 30):
+                print("In last %d days, users sent:" % (days_ago,))
+                sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
+                for quantity in sorted(sender_quantities, reverse=True):
+                    print(quantity, end=' ')
+                print("")
+
+                print("%d stream messages" % (self.stream_messages(realm, days_ago),))
+                print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
+                print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
+                print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
+
+            num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
+            self.report_percentage(num_notifications_enabled, num_active,
+                                   "active users have desktop notifications enabled")
+
+            num_enter_sends = len([x for x in active_users if x.enter_sends])
+            self.report_percentage(num_enter_sends, num_active,
+                                   "active users have enter-sends")
+
+            all_message_count = human_messages.filter(sender__realm=realm).count()
+            multi_paragraph_message_count = human_messages.filter(
+                sender__realm=realm, content__contains="\n\n").count()
+            self.report_percentage(multi_paragraph_message_count, all_message_count,
+                                   "all messages are multi-paragraph")
+
+            # Starred messages
+            starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
+                                                  flags=UserMessage.flags.starred).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users have starred %d messages" % (
+                len(starrers), sum([elt["count"] for elt in starrers])))
+
+            active_user_subs = Subscription.objects.filter(
+                user_profile__in=user_profiles, active=True)
+
+            # Streams not in home view
+            non_home_view = active_user_subs.filter(in_home_view=False).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users have %d streams not in home view" % (
+                len(non_home_view), sum([elt["count"] for elt in non_home_view])))
+
+            # Code block markup
+            markup_messages = human_messages.filter(
+                sender__realm=realm, content__contains="~~~").values(
+                "sender").annotate(count=Count("sender"))
+            print("%d users have used code block markup on %s messages" % (
+                len(markup_messages), sum([elt["count"] for elt in markup_messages])))
+
+            # Notifications for stream messages
+            notifications = active_user_subs.filter(notifications=True).values(
+                "user_profile").annotate(count=Count("user_profile"))
+            print("%d users receive desktop notifications for %d streams" % (
+                len(notifications), sum([elt["count"] for elt in notifications])))
+
+            print("")

analytics/management/commands/stream_stats.py

@@ -0,0 +1,46 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from typing import Any
+
+from argparse import ArgumentParser
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+from zerver.models import Realm, Stream, Message, Subscription, Recipient, get_realm
+
+class Command(BaseCommand):
+    help = "Generate statistics on the streams for a realm."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def handle(self, *args, **options):
+        # type: (*Any, **str) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(string_id) for string_id in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.string_id)
+            print("------------")
+            print("%25s %15s %10s" % ("stream", "subscribers", "messages"))
+            streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
+            invite_only_count = 0
+            for stream in streams:
+                if stream.invite_only:
+                    invite_only_count += 1
+                    continue
+                print("%25s" % (stream.name,), end=' ')
+                recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
+                print("%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),), end=' ')
+                num_messages = len(Message.objects.filter(recipient=recipient))
+                print("%12d" % (num_messages,))
+            print("%d invite-only streams" % (invite_only_count,))
+            print("")

analytics/management/commands/update_analytics_counts.py

@@ -1,99 +1,75 @@
-import hashlib
+from __future__ import absolute_import
-import time
+from __future__ import print_function
+
+import os
+import sys
+from scripts.lib.zulip_tools import ENDC, WARNING
+
 from argparse import ArgumentParser
-from datetime import timezone
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+from django.utils.dateparse import parse_datetime
+from django.conf import settings
+
+from analytics.models import RealmCount, UserCount
+from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
+from zerver.lib.timestamp import datetime_to_string, is_timezone_aware
+from zerver.models import UserProfile, Message
+
 from typing import Any
 
-from django.conf import settings
+class Command(BaseCommand):
-from django.utils.dateparse import parse_datetime
-from django.utils.timezone import now as timezone_now
-from typing_extensions import override
-
-from analytics.lib.counts import ALL_COUNT_STATS, logger, process_count_stat
-from zerver.lib.management import ZulipBaseCommand, abort_unless_locked
-from zerver.lib.remote_server import send_server_data_to_push_bouncer, should_send_analytics_data
-from zerver.lib.timestamp import floor_to_hour
-from zerver.models import Realm
-
-
-class Command(ZulipBaseCommand):
     help = """Fills Analytics tables.
 
     Run as a cron job that runs every hour."""
 
-    @override
+    def add_arguments(self, parser):
-    def add_arguments(self, parser: ArgumentParser) -> None:
+        # type: (ArgumentParser) -> None
-        parser.add_argument(
+        parser.add_argument('--time', '-t',
-            "--time",
+                            type=str,
-            "-t",
+                            help='Update stat tables from current state to --time. Defaults to the current time.',
-            help="Update stat tables from current state to "
+                            default=datetime_to_string(timezone.now()))
-            "--time. Defaults to the current time.",
+        parser.add_argument('--utc',
-            default=timezone_now().isoformat(),
+                            type=bool,
-        )
+                            help="Interpret --time in UTC.",
-        parser.add_argument("--utc", action="store_true", help="Interpret --time in UTC.")
+                            default=False)
-        parser.add_argument(
+        parser.add_argument('--stat', '-s',
-            "--stat", "-s", help="CountStat to process. If omitted, all stats are processed."
+                            type=str,
-        )
+                            help="CountStat to process. If omitted, all stats are processed.")
-        parser.add_argument(
+        parser.add_argument('--quiet', '-q',
-            "--verbose", action="store_true", help="Print timing information to stdout."
+                            type=str,
-        )
+                            help="Suppress output to stdout.")
 
-    @override
+    def handle(self, *args, **options):
-    @abort_unless_locked
+        # type: (*Any, **Any) -> None
-    def handle(self, *args: Any, **options: Any) -> None:
+        try:
-        self.run_update_analytics_counts(options)
+            os.mkdir(settings.ANALYTICS_LOCK_DIR)
-
+        except OSError:
-    def run_update_analytics_counts(self, options: dict[str, Any]) -> None:
+            print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
-        # installation_epoch relies on there being at least one realm; we
-        # shouldn't run the analytics code if that condition isn't satisfied
-        if not Realm.objects.exists():
-            logger.info("No realms, stopping update_analytics_counts")
             return
 
-        fill_to_time = parse_datetime(options["time"])
+        try:
-        assert fill_to_time is not None
+            self.run_update_analytics_counts(options)
-        if options["utc"]:
+        finally:
+            os.rmdir(settings.ANALYTICS_LOCK_DIR)
+
+    def run_update_analytics_counts(self, options):
+        # type: (Dict[str, Any]) -> None
+        fill_to_time = parse_datetime(options['time'])
+        if options['utc']:
             fill_to_time = fill_to_time.replace(tzinfo=timezone.utc)
-        if fill_to_time.tzinfo is None:
-            raise ValueError(
-                "--time must be time-zone-aware. Maybe you meant to use the --utc option?"
-            )
 
-        fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone.utc))
+        if not (is_timezone_aware(fill_to_time)):
+            raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
 
-        if options["stat"] is not None:
+        logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
-            stats = [ALL_COUNT_STATS[options["stat"]]]
+
+        if options['stat'] is not None:
+            process_count_stat(COUNT_STATS[options['stat']], fill_to_time)
         else:
-            stats = list(ALL_COUNT_STATS.values())
+            for stat in COUNT_STATS.values():
+                process_count_stat(stat, fill_to_time)
 
-        logger.info("Starting updating analytics counts through %s", fill_to_time)
+        logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
-        if options["verbose"]:
-            start = time.time()
-            last = start
-
-        for stat in stats:
-            process_count_stat(stat, fill_to_time)
-            if options["verbose"]:
-                print(f"Updated {stat.property} in {time.time() - last:.3f}s")
-                last = time.time()
-
-        if options["verbose"]:
-            print(
-                f"Finished updating analytics counts through {fill_to_time} in {time.time() - start:.3f}s"
-            )
-        logger.info("Finished updating analytics counts through %s", fill_to_time)
-
-        if should_send_analytics_data():
-            # Based on the specific value of the setting, the exact details to send
-            # will be decided. However, we proceed just based on this not being falsey.
-
-            # Skew 0-10 minutes based on a hash of settings.ZULIP_ORG_ID, so
-            # that each server will report in at a somewhat consistent time.
-            assert settings.ZULIP_ORG_ID
-            delay = int.from_bytes(
-                hashlib.sha256(settings.ZULIP_ORG_ID.encode()).digest(), byteorder="big"
-            ) % (60 * 10)
-            logger.info("Sleeping %d seconds before reporting...", delay)
-            time.sleep(delay)
-
-            send_server_data_to_push_bouncer(consider_usage_statistics=True)

analytics/management/commands/user_stats.py

@@ -0,0 +1,48 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+from argparse import ArgumentParser
+import datetime
+import pytz
+from typing import Any
+
+from django.core.management.base import BaseCommand
+from zerver.models import UserProfile, Realm, Stream, Message, get_realm
+from six.moves import range
+
+class Command(BaseCommand):
+    help = "Generate statistics on user activity."
+
+    def add_arguments(self, parser):
+        # type: (ArgumentParser) -> None
+        parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
+                            help="realm to generate statistics for")
+
+    def messages_sent_by(self, user, week):
+        # type: (UserProfile, int) -> int
+        start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
+        end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
+        return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
+
+    def handle(self, *args, **options):
+        # type: (*Any, **Any) -> None
+        if options['realms']:
+            try:
+                realms = [get_realm(string_id) for string_id in options['realms']]
+            except Realm.DoesNotExist as e:
+                print(e)
+                exit(1)
+        else:
+            realms = Realm.objects.all()
+
+        for realm in realms:
+            print(realm.string_id)
+            user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
+            print("%d users" % (len(user_profiles),))
+            print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
+
+            for user_profile in user_profiles:
+                print("%35s" % (user_profile.email,), end=' ')
+                for week in range(10):
+                    print("%5d" % (self.messages_sent_by(user_profile, week)), end=' ')
+                print("")

analytics/migrations/0001_initial.py

@@ -1,208 +1,113 @@
-import django.db.models.deletion
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
 from django.conf import settings
-from django.db import migrations, models
+import zerver.lib.str_utils
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("zerver", "0030_realm_org_type"),
+        ('zerver', '0030_realm_org_type'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
 
     operations = [
         migrations.CreateModel(
-            name="Anomaly",
+            name='Anomaly',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('info', models.CharField(max_length=1000)),
-                    models.AutoField(
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
-                    ),
-                ),
-                ("info", models.CharField(max_length=1000)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
-            name="HuddleCount",
+            name='HuddleCount',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('huddle', models.ForeignKey(to='zerver.Recipient')),
-                    models.AutoField(
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('property', models.CharField(max_length=40)),
-                    ),
+                ('end_time', models.DateTimeField()),
-                ),
+                ('interval', models.CharField(max_length=20)),
-                (
+                ('value', models.BigIntegerField()),
-                    "huddle",
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Recipient"
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
-            name="InstallationCount",
+            name='InstallationCount',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('property', models.CharField(max_length=40)),
-                    models.AutoField(
+                ('end_time', models.DateTimeField()),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('interval', models.CharField(max_length=20)),
-                    ),
+                ('value', models.BigIntegerField()),
-                ),
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
-            name="RealmCount",
+            name='RealmCount',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('realm', models.ForeignKey(to='zerver.Realm')),
-                    models.AutoField(
+                ('property', models.CharField(max_length=40)),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('end_time', models.DateTimeField()),
-                    ),
+                ('interval', models.CharField(max_length=20)),
-                ),
+                ('value', models.BigIntegerField()),
-                (
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-                    "realm",
+
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
-            name="StreamCount",
+            name='StreamCount',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('realm', models.ForeignKey(to='zerver.Realm')),
-                    models.AutoField(
+                ('stream', models.ForeignKey(to='zerver.Stream')),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('property', models.CharField(max_length=40)),
-                    ),
+                ('end_time', models.DateTimeField()),
-                ),
+                ('interval', models.CharField(max_length=20)),
-                (
+                ('value', models.BigIntegerField()),
-                    "realm",
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                (
-                    "stream",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Stream"
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.CreateModel(
-            name="UserCount",
+            name='UserCount',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('realm', models.ForeignKey(to='zerver.Realm')),
-                    models.AutoField(
+                ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('property', models.CharField(max_length=40)),
-                    ),
+                ('end_time', models.DateTimeField()),
-                ),
+                ('interval', models.CharField(max_length=20)),
-                (
+                ('value', models.BigIntegerField()),
-                    "realm",
+                ('anomaly', models.ForeignKey(to='analytics.Anomaly', null=True)),
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="zerver.Realm"
-                    ),
-                ),
-                (
-                    "user",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
-                    ),
-                ),
-                ("property", models.CharField(max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("interval", models.CharField(max_length=20)),
-                ("value", models.BigIntegerField()),
-                (
-                    "anomaly",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="analytics.Anomaly",
-                        null=True,
-                    ),
-                ),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
         migrations.AlterUniqueTogether(
-            name="usercount",
+            name='usercount',
-            unique_together={("user", "property", "end_time", "interval")},
+            unique_together=set([('user', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="streamcount",
+            name='streamcount',
-            unique_together={("stream", "property", "end_time", "interval")},
+            unique_together=set([('stream', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="realmcount",
+            name='realmcount',
-            unique_together={("realm", "property", "end_time", "interval")},
+            unique_together=set([('realm', 'property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="installationcount",
+            name='installationcount',
-            unique_together={("property", "end_time", "interval")},
+            unique_together=set([('property', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="huddlecount",
+            name='huddlecount',
-            unique_together={("huddle", "property", "end_time", "interval")},
+            unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
         ),
     ]

analytics/migrations/0002_remove_huddlecount.py

@@ -1,29 +1,33 @@
-from django.db import migrations
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0001_initial"),
+        ('analytics', '0001_initial'),
     ]
 
     operations = [
         migrations.AlterUniqueTogether(
-            name="huddlecount",
+            name='huddlecount',
-            unique_together=set(),
+            unique_together=set([]),
         ),
         migrations.RemoveField(
-            model_name="huddlecount",
+            model_name='huddlecount',
-            name="anomaly",
+            name='anomaly',
         ),
         migrations.RemoveField(
-            model_name="huddlecount",
+            model_name='huddlecount',
-            name="huddle",
+            name='huddle',
         ),
         migrations.RemoveField(
-            model_name="huddlecount",
+            model_name='huddlecount',
-            name="user",
+            name='user',
         ),
         migrations.DeleteModel(
-            name="HuddleCount",
+            name='HuddleCount',
         ),
     ]

analytics/migrations/0003_fillstate.py

@@ -1,26 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
+import zerver.lib.str_utils
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0002_remove_huddlecount"),
+        ('analytics', '0002_remove_huddlecount'),
     ]
 
     operations = [
         migrations.CreateModel(
-            name="FillState",
+            name='FillState',
             fields=[
-                (
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                    "id",
+                ('property', models.CharField(unique=True, max_length=40)),
-                    models.AutoField(
+                ('end_time', models.DateTimeField()),
-                        verbose_name="ID", serialize=False, auto_created=True, primary_key=True
+                ('state', models.PositiveSmallIntegerField()),
-                    ),
+                ('last_modified', models.DateTimeField(auto_now=True)),
-                ),
-                ("property", models.CharField(unique=True, max_length=40)),
-                ("end_time", models.DateTimeField()),
-                ("state", models.PositiveSmallIntegerField()),
-                ("last_modified", models.DateTimeField(auto_now=True)),
             ],
-            bases=(models.Model,),
+            bases=(zerver.lib.str_utils.ModelReprMixin, models.Model),
         ),
     ]

analytics/migrations/0004_add_subgroup.py

@@ -1,30 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0003_fillstate"),
+        ('analytics', '0003_fillstate'),
     ]
 
     operations = [
         migrations.AddField(
-            model_name="installationcount",
+            model_name='installationcount',
-            name="subgroup",
+            name='subgroup',
             field=models.CharField(max_length=16, null=True),
         ),
         migrations.AddField(
-            model_name="realmcount",
+            model_name='realmcount',
-            name="subgroup",
+            name='subgroup',
             field=models.CharField(max_length=16, null=True),
         ),
         migrations.AddField(
-            model_name="streamcount",
+            model_name='streamcount',
-            name="subgroup",
+            name='subgroup',
             field=models.CharField(max_length=16, null=True),
         ),
         migrations.AddField(
-            model_name="usercount",
+            model_name='usercount',
-            name="subgroup",
+            name='subgroup',
             field=models.CharField(max_length=16, null=True),
         ),
     ]

analytics/migrations/0005_alter_field_size.py

@@ -1,50 +1,54 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0004_add_subgroup"),
+        ('analytics', '0004_add_subgroup'),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name="installationcount",
+            model_name='installationcount',
-            name="interval",
+            name='interval',
             field=models.CharField(max_length=8),
         ),
         migrations.AlterField(
-            model_name="installationcount",
+            model_name='installationcount',
-            name="property",
+            name='property',
             field=models.CharField(max_length=32),
         ),
         migrations.AlterField(
-            model_name="realmcount",
+            model_name='realmcount',
-            name="interval",
+            name='interval',
             field=models.CharField(max_length=8),
         ),
         migrations.AlterField(
-            model_name="realmcount",
+            model_name='realmcount',
-            name="property",
+            name='property',
             field=models.CharField(max_length=32),
         ),
         migrations.AlterField(
-            model_name="streamcount",
+            model_name='streamcount',
-            name="interval",
+            name='interval',
             field=models.CharField(max_length=8),
         ),
         migrations.AlterField(
-            model_name="streamcount",
+            model_name='streamcount',
-            name="property",
+            name='property',
             field=models.CharField(max_length=32),
         ),
         migrations.AlterField(
-            model_name="usercount",
+            model_name='usercount',
-            name="interval",
+            name='interval',
             field=models.CharField(max_length=8),
         ),
         migrations.AlterField(
-            model_name="usercount",
+            model_name='usercount',
-            name="property",
+            name='property',
             field=models.CharField(max_length=32),
         ),
     ]

analytics/migrations/0006_add_subgroup_to_unique_constraints.py

@@ -1,26 +1,30 @@
-from django.db import migrations
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0005_alter_field_size"),
+        ('analytics', '0005_alter_field_size'),
     ]
 
     operations = [
         migrations.AlterUniqueTogether(
-            name="installationcount",
+            name='installationcount',
-            unique_together={("property", "subgroup", "end_time", "interval")},
+            unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="realmcount",
+            name='realmcount',
-            unique_together={("realm", "property", "subgroup", "end_time", "interval")},
+            unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="streamcount",
+            name='streamcount',
-            unique_together={("stream", "property", "subgroup", "end_time", "interval")},
+            unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
         migrations.AlterUniqueTogether(
-            name="usercount",
+            name='usercount',
-            unique_together={("user", "property", "subgroup", "end_time", "interval")},
+            unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
         ),
     ]

analytics/migrations/0007_remove_interval.py

@@ -1,43 +1,48 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.10.4 on 2017-01-16 20:50
+from __future__ import unicode_literals
+
+from django.conf import settings
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("analytics", "0006_add_subgroup_to_unique_constraints"),
+        ('analytics', '0006_add_subgroup_to_unique_constraints'),
     ]
 
     operations = [
         migrations.AlterUniqueTogether(
-            name="installationcount",
+            name='installationcount',
-            unique_together={("property", "subgroup", "end_time")},
+            unique_together=set([('property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
-            model_name="installationcount",
+            model_name='installationcount',
-            name="interval",
+            name='interval',
         ),
         migrations.AlterUniqueTogether(
-            name="realmcount",
+            name='realmcount',
-            unique_together={("realm", "property", "subgroup", "end_time")},
+            unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
-            model_name="realmcount",
+            model_name='realmcount',
-            name="interval",
+            name='interval',
         ),
         migrations.AlterUniqueTogether(
-            name="streamcount",
+            name='streamcount',
-            unique_together={("stream", "property", "subgroup", "end_time")},
+            unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
-            model_name="streamcount",
+            model_name='streamcount',
-            name="interval",
+            name='interval',
         ),
         migrations.AlterUniqueTogether(
-            name="usercount",
+            name='usercount',
-            unique_together={("user", "property", "subgroup", "end_time")},
+            unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
         ),
         migrations.RemoveField(
-            model_name="usercount",
+            model_name='usercount',
-            name="interval",
+            name='interval',
         ),
     ]

analytics/migrations/0008_add_count_indexes.py

@@ -1,33 +1,28 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 1.10.5 on 2017-02-01 22:28
-from django.db import migrations, models
+from __future__ import unicode_literals
+
+from django.db import migrations
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
-        ("zerver", "0050_userprofile_avatar_version"),
+        ('zerver', '0050_userprofile_avatar_version'),
-        ("analytics", "0007_remove_interval"),
+        ('analytics', '0007_remove_interval'),
     ]
 
     operations = [
-        migrations.AddIndex(
+        migrations.AlterIndexTogether(
-            model_name="realmcount",
+            name='realmcount',
-            index=models.Index(
+            index_together=set([('property', 'end_time')]),
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            ),
         ),
-        migrations.AddIndex(
+        migrations.AlterIndexTogether(
-            model_name="streamcount",
+            name='streamcount',
-            index=models.Index(
+            index_together=set([('property', 'realm', 'end_time')]),
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            ),
         ),
-        migrations.AddIndex(
+        migrations.AlterIndexTogether(
-            model_name="usercount",
+            name='usercount',
-            index=models.Index(
+            index_together=set([('property', 'realm', 'end_time')]),
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            ),
         ),
     ]

analytics/migrations/0009_remove_messages_to_stream_stat.py

@@ -1,30 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def delete_messages_sent_to_stream_stat(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    property = "messages_sent_to_stream:is_bot"
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0008_add_count_indexes"),
-    ]
-
-    operations = [
-        migrations.RunPython(delete_messages_sent_to_stream_stat),
-    ]

analytics/migrations/0010_clear_messages_sent_values.py

@@ -1,28 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def clear_message_sent_by_message_type_values(
-    apps: StateApps, schema_editor: BaseDatabaseSchemaEditor
-) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    property = "messages_sent:message_type:day"
-    UserCount.objects.filter(property=property).delete()
-    StreamCount.objects.filter(property=property).delete()
-    RealmCount.objects.filter(property=property).delete()
-    InstallationCount.objects.filter(property=property).delete()
-    FillState.objects.filter(property=property).delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [("analytics", "0009_remove_messages_to_stream_stat")]
-
-    operations = [
-        migrations.RunPython(clear_message_sent_by_message_type_values),
-    ]

analytics/migrations/0011_clear_analytics_tables.py

@@ -1,27 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-
-
-def clear_analytics_tables(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
-    UserCount = apps.get_model("analytics", "UserCount")
-    StreamCount = apps.get_model("analytics", "StreamCount")
-    RealmCount = apps.get_model("analytics", "RealmCount")
-    InstallationCount = apps.get_model("analytics", "InstallationCount")
-    FillState = apps.get_model("analytics", "FillState")
-
-    UserCount.objects.all().delete()
-    StreamCount.objects.all().delete()
-    RealmCount.objects.all().delete()
-    InstallationCount.objects.all().delete()
-    FillState.objects.all().delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0010_clear_messages_sent_values"),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_analytics_tables),
-    ]

analytics/migrations/0012_add_on_delete.py

@@ -1,41 +0,0 @@
-# Generated by Django 1.11.6 on 2018-01-29 08:14
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0011_clear_analytics_tables"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="anomaly",
-            field=models.ForeignKey(
-                null=True, on_delete=django.db.models.deletion.SET_NULL, to="analytics.Anomaly"
-            ),
-        ),
-    ]

analytics/migrations/0013_remove_anomaly.py

@@ -1,31 +0,0 @@
-# Generated by Django 1.11.18 on 2019-02-02 02:47
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0012_add_on_delete"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="installationcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="realmcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="streamcount",
-            name="anomaly",
-        ),
-        migrations.RemoveField(
-            model_name="usercount",
-            name="anomaly",
-        ),
-        migrations.DeleteModel(
-            name="Anomaly",
-        ),
-    ]

analytics/migrations/0014_remove_fillstate_last_modified.py

@@ -1,16 +0,0 @@
-# Generated by Django 1.11.26 on 2020-01-27 04:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0013_remove_anomaly"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="fillstate",
-            name="last_modified",
-        ),
-    ]

analytics/migrations/0015_clear_duplicate_counts.py

@@ -1,64 +0,0 @@
-from django.db import migrations
-from django.db.backends.base.schema import BaseDatabaseSchemaEditor
-from django.db.migrations.state import StateApps
-from django.db.models import Count, Sum
-
-
-def clear_duplicate_counts(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None:
-    """This is a preparatory migration for our Analytics tables.
-
-    The backstory is that Django's unique_together indexes do not properly
-    handle the subgroup=None corner case (allowing duplicate rows that have a
-    subgroup of None), which meant that in race conditions, rather than updating
-    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
-    create a duplicate row.
-
-    In the next migration, we'll add a proper constraint to fix this bug, but
-    we need to fix any existing problematic rows before we can add that constraint.
-
-    We fix this in an appropriate fashion for each type of CountStat object; mainly
-    this means deleting the extra rows, but for LoggingCountStat objects, we need to
-    additionally combine the sums.
-    """
-    count_tables = dict(
-        realm=apps.get_model("analytics", "RealmCount"),
-        user=apps.get_model("analytics", "UserCount"),
-        stream=apps.get_model("analytics", "StreamCount"),
-        installation=apps.get_model("analytics", "InstallationCount"),
-    )
-
-    for name, count_table in count_tables.items():
-        value = [name, "property", "end_time"]
-        if name == "installation":
-            value = ["property", "end_time"]
-        counts = (
-            count_table.objects.filter(subgroup=None)
-            .values(*value)
-            .annotate(Count("id"), Sum("value"))
-            .filter(id__count__gt=1)
-        )
-
-        for count in counts:
-            count.pop("id__count")
-            total_value = count.pop("value__sum")
-            duplicate_counts = list(count_table.objects.filter(**count))
-            first_count = duplicate_counts[0]
-            if count["property"] in ["invites_sent::day", "active_users_log:is_bot:day"]:
-                # For LoggingCountStat objects, the right fix is to combine the totals;
-                # for other CountStat objects, we expect the duplicates to have the same value.
-                # And so all we need to do is delete them.
-                first_count.value = total_value
-                first_count.save()
-            to_cleanup = duplicate_counts[1:]
-            for duplicate_count in to_cleanup:
-                duplicate_count.delete()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0014_remove_fillstate_last_modified"),
-    ]
-
-    operations = [
-        migrations.RunPython(clear_duplicate_counts, reverse_code=migrations.RunPython.noop),
-    ]

analytics/migrations/0016_unique_constraint_when_subgroup_null.py

@@ -1,92 +0,0 @@
-# Generated by Django 2.2.10 on 2020-02-29 19:40
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0015_clear_duplicate_counts"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="installationcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="realmcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="streamcount",
-            unique_together=set(),
-        ),
-        migrations.AlterUniqueTogether(
-            name="usercount",
-            unique_together=set(),
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/migrations/0017_regenerate_partial_indexes.py

@@ -1,114 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0016_unique_constraint_when_subgroup_null"),
-    ]
-
-    # If the server was installed between 7.0 and 7.4 (or main between
-    # 2c20028aa451 and 7807bff52635), it contains indexes which (when
-    # running 7.5 or 7807bff52635 or higher) are never used, because
-    # they contain an improper cast
-    # (https://code.djangoproject.com/ticket/34840).
-    #
-    # We regenerate the indexes here, by dropping and re-creating
-    # them, so that we know that they are properly formed.
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("property", "subgroup", "end_time"),
-                name="unique_installation_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="installationcount",
-            name="unique_installation_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="installationcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("property", "end_time"),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("realm", "property", "subgroup", "end_time"),
-                name="unique_realm_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="realmcount",
-            name="unique_realm_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="realmcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("realm", "property", "end_time"),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("stream", "property", "subgroup", "end_time"),
-                name="unique_stream_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="streamcount",
-            name="unique_stream_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="streamcount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("stream", "property", "end_time"),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=False),
-                fields=("user", "property", "subgroup", "end_time"),
-                name="unique_user_count",
-            ),
-        ),
-        migrations.RemoveConstraint(
-            model_name="usercount",
-            name="unique_user_count_null_subgroup",
-        ),
-        migrations.AddConstraint(
-            model_name="usercount",
-            constraint=models.UniqueConstraint(
-                condition=models.Q(subgroup__isnull=True),
-                fields=("user", "property", "end_time"),
-                name="unique_user_count_null_subgroup",
-            ),
-        ),
-    ]

analytics/migrations/0018_remove_usercount_active_users_audit.py

@@ -1,15 +0,0 @@
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0017_regenerate_partial_indexes"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            "DELETE FROM analytics_usercount WHERE property = 'active_users_audit:is_bot:day'"
-        )
-    ]

analytics/migrations/0019_remove_unused_counts.py

@@ -1,26 +0,0 @@
-from django.db import migrations
-
-REMOVED_COUNTS = (
-    "active_users_log:is_bot:day",
-    "active_users:is_bot:day",
-)
-
-
-class Migration(migrations.Migration):
-    elidable = True
-
-    dependencies = [
-        ("analytics", "0018_remove_usercount_active_users_audit"),
-    ]
-
-    operations = [
-        migrations.RunSQL(
-            [
-                ("DELETE FROM analytics_realmcount WHERE property IN %s", (REMOVED_COUNTS,)),
-                (
-                    "DELETE FROM analytics_installationcount WHERE property IN %s",
-                    (REMOVED_COUNTS,),
-                ),
-            ]
-        )
-    ]

analytics/migrations/0020_alter_installationcount_id_alter_realmcount_id_and_more.py

@@ -1,40 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    atomic = False
-
-    dependencies = [
-        ("analytics", "0019_remove_unused_counts"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="installationcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="realmcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="streamcount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-        migrations.AlterField(
-            model_name="usercount",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/migrations/0021_alter_fillstate_id.py

@@ -1,17 +0,0 @@
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("analytics", "0020_alter_installationcount_id_alter_realmcount_id_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="fillstate",
-            name="id",
-            field=models.BigAutoField(
-                auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
-            ),
-        ),
-    ]

analytics/models.py

@@ -1,161 +1,151 @@
-# https://github.com/typeddjango/django-stubs/issues/1698
-# mypy: disable-error-code="explicit-override"
-
-from datetime import datetime
-
 from django.db import models
-from django.db.models import Q, UniqueConstraint
+from django.utils import timezone
-from typing_extensions import override
 
-from zerver.lib.timestamp import floor_to_day
+from zerver.models import Realm, UserProfile, Stream, Recipient
-from zerver.models import Realm, Stream, UserProfile
+from zerver.lib.str_utils import ModelReprMixin
+from zerver.lib.timestamp import datetime_to_UTC, floor_to_day
 
+import datetime
 
-class FillState(models.Model):
+from typing import Optional, Tuple, Union, Dict, Any, Text
-    property = models.CharField(max_length=40, unique=True)
+
-    end_time = models.DateTimeField()
+class FillState(ModelReprMixin, models.Model):
+    property = models.CharField(max_length=40, unique=True) # type: Text
+    end_time = models.DateTimeField() # type: datetime.datetime
 
     # Valid states are {DONE, STARTED}
     DONE = 1
     STARTED = 2
-    state = models.PositiveSmallIntegerField()
+    state = models.PositiveSmallIntegerField() # type: int
 
-    @override
+    last_modified = models.DateTimeField(auto_now=True) # type: datetime.datetime
-    def __str__(self) -> str:
-        return f"{self.property} {self.end_time} {self.state}"
 
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
 
 # The earliest/starting end_time in FillState
 # We assume there is at least one realm
-def installation_epoch() -> datetime:
+def installation_epoch():
-    earliest_realm_creation = Realm.objects.aggregate(models.Min("date_created"))[
+    # type: () -> datetime.datetime
-        "date_created__min"
+    earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
-    ]
+    return floor_to_day(datetime_to_UTC(earliest_realm_creation))
-    return floor_to_day(earliest_realm_creation)
 
+# would only ever make entries here by hand
+class Anomaly(ModelReprMixin, models.Model):
+    info = models.CharField(max_length=1000) # type: Text
 
-class BaseCount(models.Model):
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<Anomaly: %s... %s>" % (self.info, self.id)
+
+class BaseCount(ModelReprMixin, models.Model):
     # Note: When inheriting from BaseCount, you may want to rearrange
     # the order of the columns in the migration to make sure they
     # match how you'd like the table to be arranged.
-    property = models.CharField(max_length=32)
+    property = models.CharField(max_length=32) # type: Text
-    subgroup = models.CharField(max_length=16, null=True)
+    subgroup = models.CharField(max_length=16, null=True) # type: Text
-    end_time = models.DateTimeField()
+    end_time = models.DateTimeField() # type: datetime.datetime
-    value = models.BigIntegerField()
+    value = models.BigIntegerField() # type: int
+    anomaly = models.ForeignKey(Anomaly, null=True) # type: Optional[Anomaly]
 
-    class Meta:
+    class Meta(object):
         abstract = True
 
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        raise NotImplementedError
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        raise NotImplementedError
 
 class InstallationCount(BaseCount):
-    class Meta:
-        # Handles invalid duplicate InstallationCount data
-        constraints = [
-            UniqueConstraint(
-                fields=["property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_installation_count",
-            ),
-            UniqueConstraint(
-                fields=["property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_installation_count_null_subgroup",
-            ),
-        ]
 
-    @override
+    class Meta(object):
-    def __str__(self) -> str:
+        unique_together = ("property", "subgroup", "end_time")
-        return f"{self.property} {self.subgroup} {self.value}"
 
+    @staticmethod
+    def extended_id():
+        # type: () -> Tuple[str, ...]
+        return ()
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return None
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
 
 class RealmCount(BaseCount):
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
-        # Handles invalid duplicate RealmCount data
+        unique_together = ("realm", "property", "subgroup", "end_time")
-        constraints = [
+        index_together = ["property", "end_time"]
-            UniqueConstraint(
-                fields=["realm", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_realm_count",
-            ),
-            UniqueConstraint(
-                fields=["realm", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_realm_count_null_subgroup",
-            ),
-        ]
-        indexes = [
-            models.Index(
-                fields=["property", "end_time"],
-                name="analytics_realmcount_property_end_time_3b60396b_idx",
-            )
-        ]
 
-    @override
+    @staticmethod
-    def __str__(self) -> str:
+    def extended_id():
-        return f"{self.realm!r} {self.property} {self.subgroup} {self.value}"
+        # type: () -> Tuple[str, ...]
+        return ('realm_id',)
 
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Realm
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
 
 class UserCount(BaseCount):
-    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+    user = models.ForeignKey(UserProfile)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
-        # Handles invalid duplicate UserCount data
+        unique_together = ("user", "property", "subgroup", "end_time")
-        constraints = [
-            UniqueConstraint(
-                fields=["user", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_user_count",
-            ),
-            UniqueConstraint(
-                fields=["user", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_user_count_null_subgroup",
-            ),
-        ]
         # This index dramatically improves the performance of
         # aggregating from users to realms
-        indexes = [
+        index_together = ["property", "realm", "end_time"]
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_usercount_property_realm_id_end_time_591dbec1_idx",
-            )
-        ]
 
-    @override
+    @staticmethod
-    def __str__(self) -> str:
+    def extended_id():
-        return f"{self.user!r} {self.property} {self.subgroup} {self.value}"
+        # type: () -> Tuple[str, ...]
+        return ('user_id', 'realm_id')
 
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return UserProfile
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
 
 class StreamCount(BaseCount):
-    stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
+    stream = models.ForeignKey(Stream)
-    realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
+    realm = models.ForeignKey(Realm)
 
-    class Meta:
+    class Meta(object):
-        # Handles invalid duplicate StreamCount data
+        unique_together = ("stream", "property", "subgroup", "end_time")
-        constraints = [
-            UniqueConstraint(
-                fields=["stream", "property", "subgroup", "end_time"],
-                condition=Q(subgroup__isnull=False),
-                name="unique_stream_count",
-            ),
-            UniqueConstraint(
-                fields=["stream", "property", "end_time"],
-                condition=Q(subgroup__isnull=True),
-                name="unique_stream_count_null_subgroup",
-            ),
-        ]
         # This index dramatically improves the performance of
         # aggregating from streams to realms
-        indexes = [
+        index_together = ["property", "realm", "end_time"]
-            models.Index(
-                fields=["property", "realm", "end_time"],
-                name="analytics_streamcount_property_realm_id_end_time_155ae930_idx",
-            )
-        ]
 
-    @override
+    @staticmethod
-    def __str__(self) -> str:
+    def extended_id():
-        return f"{self.stream!r} {self.property} {self.subgroup} {self.value} {self.id}"
+        # type: () -> Tuple[str, ...]
+        return ('stream_id', 'realm_id')
+
+    @staticmethod
+    def key_model():
+        # type: () -> models.Model
+        return Stream
+
+    def __unicode__(self):
+        # type: () -> Text
+        return u"<StreamCount: %s %s %s %s %s>" % (self.stream, self.property, self.subgroup, self.value, self.id)

analytics/tests/test_fixtures.py

@@ -1,40 +0,0 @@
-from analytics.lib.counts import CountStat
-from analytics.lib.fixtures import generate_time_series_data
-from zerver.lib.test_classes import ZulipTestCase
-
-
-# A very light test suite; the code being tested is not run in production.
-class TestFixtures(ZulipTestCase):
-    def test_deterministic_settings(self) -> None:
-        # test basic business_hour / non_business_hour calculation
-        # test we get an array of the right length with frequency=CountStat.DAY
-        data = generate_time_series_data(
-            days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0
-        )
-        self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
-
-        data = generate_time_series_data(
-            days=1,
-            business_hours_base=2000,
-            non_business_hours_base=1500,
-            growth=2,
-            spikiness=0,
-            frequency=CountStat.HOUR,
-        )
-        # test we get an array of the right length with frequency=CountStat.HOUR
-        self.assert_length(data, 24)
-        # test that growth doesn't affect the first data point
-        self.assertEqual(data[0], 2000)
-        # test that the last data point is growth times what it otherwise would be
-        self.assertEqual(data[-1], 1500 * 2)
-
-        # test autocorrelation == 1, since that's the easiest value to test
-        data = generate_time_series_data(
-            days=1,
-            business_hours_base=2000,
-            non_business_hours_base=2000,
-            autocorrelation=1,
-            frequency=CountStat.HOUR,
-        )
-        self.assertEqual(data[0], data[1])
-        self.assertEqual(data[0], data[-1])

analytics/tests/test_stats_views.py

@@ -1,689 +0,0 @@
-from datetime import datetime, timedelta, timezone
-
-from django.utils.timezone import now as timezone_now
-from typing_extensions import override
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import FillState, RealmCount, StreamCount, UserCount
-from analytics.views.stats import rewrite_client_arrays, sort_by_totals, sort_client_labels
-from zerver.lib.test_classes import ZulipTestCase
-from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp
-from zerver.models import Client
-from zerver.models.realms import get_realm
-
-
-class TestStatsEndpoint(ZulipTestCase):
-    def test_stats(self) -> None:
-        self.user = self.example_user("hamlet")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assertEqual(result.status_code, 200)
-        # Check that we get something back
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_guest_user_cant_access_stats(self) -> None:
-        self.user = self.example_user("polonius")
-        self.login_user(self.user)
-        result = self.client_get("/stats")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-        result = self.client_get("/json/analytics/chart_data")
-        self.assert_json_error(result, "Not allowed for guest users", 400)
-
-    def test_stats_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 302)
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/realm/not_existing_realm/")
-        self.assertEqual(result.status_code, 404)
-
-        result = self.client_get("/stats/realm/zulip/")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-    def test_stats_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 302)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-
-        result = self.client_get("/stats/installation")
-        self.assertEqual(result.status_code, 200)
-        self.assert_in_response("Zulip analytics for", result)
-
-
-class TestGetChartData(ZulipTestCase):
-    @override
-    def setUp(self) -> None:
-        super().setUp()
-        self.realm = get_realm("zulip")
-        self.user = self.example_user("hamlet")
-        self.stream_id = self.get_stream_id(self.get_streams(self.user)[0])
-        self.login_user(self.user)
-        self.end_times_hour = [
-            ceiling_to_hour(self.realm.date_created) + timedelta(hours=i) for i in range(4)
-        ]
-        self.end_times_day = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(4)
-        ]
-
-    def data(self, i: int) -> list[int]:
-        return [0, 0, i, 0]
-
-    def insert_data(
-        self, stat: CountStat, realm_subgroups: list[str | None], user_subgroups: list[str]
-    ) -> None:
-        if stat.frequency == CountStat.HOUR:
-            insert_time = self.end_times_hour[2]
-            fill_time = self.end_times_hour[-1]
-        if stat.frequency == CountStat.DAY:
-            insert_time = self.end_times_day[2]
-            fill_time = self.end_times_day[-1]
-
-        RealmCount.objects.bulk_create(
-            RealmCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
-        UserCount.objects.bulk_create(
-            UserCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=200 + i,
-                realm=self.realm,
-                user=self.user,
-            )
-            for i, subgroup in enumerate(user_subgroups)
-        )
-        StreamCount.objects.bulk_create(
-            StreamCount(
-                property=stat.property,
-                subgroup=subgroup,
-                end_time=insert_time,
-                value=100 + i,
-                stream_id=self.stream_id,
-                realm=self.realm,
-            )
-            for i, subgroup in enumerate(realm_subgroups)
-        )
-        FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
-
-    def test_number_of_humans(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "_1day": self.data(100),
-                    "_15day": self.data(100),
-                    "all_time": self.data(100),
-                },
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_over_time(self) -> None:
-        stat = COUNT_STATS["messages_sent:is_bot:hour"]
-        self.insert_data(stat, ["true", "false"], ["false"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "user": {"bot": self.data(0), "human": self.data(200)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_message_type(self) -> None:
-        stat = COUNT_STATS["messages_sent:message_type:day"]
-        self.insert_data(
-            stat, ["public_stream", "private_message"], ["public_stream", "private_stream"]
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "Public channels": self.data(100),
-                    "Private channels": self.data(0),
-                    "Direct messages": self.data(101),
-                    "Group direct messages": self.data(0),
-                },
-                "user": {
-                    "Public channels": self.data(200),
-                    "Private channels": self.data(201),
-                    "Direct messages": self.data(0),
-                    "Group direct messages": self.data(0),
-                },
-                "display_order": [
-                    "Direct messages",
-                    "Public channels",
-                    "Private channels",
-                    "Group direct messages",
-                ],
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_client(self) -> None:
-        stat = COUNT_STATS["messages_sent:client:day"]
-        client1 = Client.objects.create(name="client 1")
-        client2 = Client.objects.create(name="client 2")
-        client3 = Client.objects.create(name="client 3")
-        client4 = Client.objects.create(name="client 4")
-        self.insert_data(
-            stat,
-            [str(client4.id), str(client3.id), str(client2.id)],
-            [str(client3.id), str(client1.id)],
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {
-                    "client 4": self.data(100),
-                    "client 3": self.data(101),
-                    "client 2": self.data(102),
-                },
-                "user": {"client 3": self.data(200), "client 1": self.data(201)},
-                "display_order": ["client 1", "client 2", "client 3", "client 4"],
-                "result": "success",
-            },
-        )
-
-    def test_messages_read_over_time(self) -> None:
-        stat = COUNT_STATS["messages_read::hour"]
-        self.insert_data(stat, [None], [])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_read_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_hour],
-                "frequency": CountStat.HOUR,
-                "everyone": {"read": self.data(100)},
-                "user": {"read": self.data(0)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-    def test_messages_sent_by_stream(self) -> None:
-        stat = COUNT_STATS["messages_in_stream:is_bot:day"]
-        self.insert_data(stat, ["true", "false"], [])
-
-        result = self.client_get(
-            f"/json/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data,
-            {
-                "msg": "",
-                "end_times": [datetime_to_timestamp(dt) for dt in self.end_times_day],
-                "frequency": CountStat.DAY,
-                "everyone": {"bot": self.data(100), "human": self.data(101)},
-                "display_order": None,
-                "result": "success",
-            },
-        )
-
-        result = self.api_get(
-            self.example_user("polonius"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-        )
-        self.assert_json_error(result, "Not allowed for guest users")
-
-        # Verify we correctly forbid access to stats of streams in other realms.
-        result = self.api_get(
-            self.mit_user("sipbtest"),
-            f"/api/v1/analytics/chart_data/stream/{self.stream_id}",
-            {
-                "chart_name": "messages_sent_by_stream",
-            },
-            subdomain="zephyr",
-        )
-        self.assert_json_error(result, "Invalid channel ID")
-
-    def test_include_empty_subgroups(self) -> None:
-        FillState.objects.create(
-            property="realm_active_humans::day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "number_of_humans"})
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"_1day": [0], "_15day": [0], "all_time": [0]})
-        self.assertFalse("user" in data)
-
-        FillState.objects.create(
-            property="messages_sent:is_bot:hour",
-            end_time=self.end_times_hour[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {"human": [0], "bot": [0]})
-        self.assertEqual(data["user"], {"human": [0], "bot": [0]})
-
-        FillState.objects.create(
-            property="messages_sent:message_type:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_message_type"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["everyone"],
-            {
-                "Public channels": [0],
-                "Private channels": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-        self.assertEqual(
-            data["user"],
-            {
-                "Public channels": [0],
-                "Private channels": [0],
-                "Direct messages": [0],
-                "Group direct messages": [0],
-            },
-        )
-
-        FillState.objects.create(
-            property="messages_sent:client:day",
-            end_time=self.end_times_day[0],
-            state=FillState.DONE,
-        )
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_by_client"}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["everyone"], {})
-        self.assertEqual(data["user"], {})
-
-    def test_start_and_end(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
-
-        # valid start and end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[1],
-                "end": end_time_timestamps[2],
-            },
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(data["end_times"], end_time_timestamps[1:3])
-        self.assertEqual(
-            data["everyone"], {"_1day": [0, 100], "_15day": [0, 100], "all_time": [0, 100]}
-        )
-
-        # start later then end
-        result = self.client_get(
-            "/json/analytics/chart_data",
-            {
-                "chart_name": "number_of_humans",
-                "start": end_time_timestamps[2],
-                "end": end_time_timestamps[1],
-            },
-        )
-        self.assert_json_error_contains(result, "Start time is later than")
-
-    def test_min_length(self) -> None:
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["1day_actives::day"]
-        self.insert_data(stat, [None], [])
-        stat = COUNT_STATS["active_users_audit:is_bot:day"]
-        self.insert_data(stat, ["false"], [])
-        # test min_length is too short to change anything
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 2}
-        )
-        data = self.assert_json_success(result)
-        self.assertEqual(
-            data["end_times"], [datetime_to_timestamp(dt) for dt in self.end_times_day]
-        )
-        self.assertEqual(
-            data["everyone"],
-            {"_1day": self.data(100), "_15day": self.data(100), "all_time": self.data(100)},
-        )
-        # test min_length larger than filled data
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "number_of_humans", "min_length": 5}
-        )
-        data = self.assert_json_success(result)
-        end_times = [
-            ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)
-        ]
-        self.assertEqual(data["end_times"], [datetime_to_timestamp(dt) for dt in end_times])
-        self.assertEqual(
-            data["everyone"],
-            {
-                "_1day": [0, *self.data(100)],
-                "_15day": [0, *self.data(100)],
-                "all_time": [0, *self.data(100)],
-            },
-        )
-
-    def test_non_existent_chart(self) -> None:
-        result = self.client_get("/json/analytics/chart_data", {"chart_name": "does_not_exist"})
-        self.assert_json_error_contains(result, "Unknown chart name")
-
-    def test_analytics_not_running(self) -> None:
-        realm = get_realm("zulip")
-
-        self.assertEqual(FillState.objects.count(), 0)
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: 0001-01-01 00:00:00+00:00 (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(hours=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=5)
-        fill_state = FillState.objects.create(
-            property="messages_sent:is_bot:hour", end_time=end_time, state=FillState.DONE
-        )
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        end_time = timezone_now() - timedelta(days=2)
-        fill_state.end_time = end_time
-        fill_state.save(update_fields=["end_time"])
-
-        realm.date_created = timezone_now() - timedelta(days=3)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-        realm.date_created = timezone_now() - timedelta(days=1, hours=2)
-        realm.save(update_fields=["date_created"])
-        with self.assertLogs(level="WARNING") as m:
-            result = self.client_get(
-                "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-            )
-            self.assertEqual(
-                m.output,
-                [
-                    f"WARNING:root:User from realm zulip attempted to access /stats, but the computed start time: {realm.date_created} (creation of realm or installation) is later than the computed end time: {end_time} (last successful analytics update). Is the analytics cron job running?"
-                ],
-            )
-
-        self.assert_json_error_contains(result, "No analytics data available")
-
-        realm.date_created = timezone_now() - timedelta(days=1, minutes=10)
-        realm.save(update_fields=["date_created"])
-        result = self.client_get(
-            "/json/analytics/chart_data", {"chart_name": "messages_sent_over_time"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_realm(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/not_existing_realm",
-            {"chart_name": "number_of_humans"},
-        )
-        self.assert_json_error(result, "Invalid organization", 400)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/realm/zulip", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-    def test_get_chart_data_for_installation(self) -> None:
-        user = self.example_user("hamlet")
-        self.login_user(user)
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_error(result, "Must be an server administrator", 400)
-
-        user = self.example_user("hamlet")
-        user.is_staff = True
-        user.save(update_fields=["is_staff"])
-        stat = COUNT_STATS["realm_active_humans::day"]
-        self.insert_data(stat, [None], [])
-
-        result = self.client_get(
-            "/json/analytics/chart_data/installation", {"chart_name": "number_of_humans"}
-        )
-        self.assert_json_success(result)
-
-
-class TestGetChartDataHelpers(ZulipTestCase):
-    def test_sort_by_totals(self) -> None:
-        empty: list[int] = []
-        value_arrays = {"c": [0, 1], "a": [9], "b": [1, 1, 1], "d": empty}
-        self.assertEqual(sort_by_totals(value_arrays), ["a", "b", "c", "d"])
-
-    def test_sort_client_labels(self) -> None:
-        data = {
-            "everyone": {"a": [16], "c": [15], "b": [14], "e": [13], "d": [12], "h": [11]},
-            "user": {"a": [6], "b": [5], "d": [4], "e": [3], "f": [2], "g": [1]},
-        }
-        self.assertEqual(sort_client_labels(data), ["a", "b", "c", "d", "e", "f", "g", "h"])
-
-
-class TestTimeRange(ZulipTestCase):
-    def test_time_range(self) -> None:
-        HOUR = timedelta(hours=1)
-        DAY = timedelta(days=1)
-
-        a_time = datetime(2016, 3, 14, 22, 59, tzinfo=timezone.utc)
-        floor_hour = datetime(2016, 3, 14, 22, tzinfo=timezone.utc)
-        floor_day = datetime(2016, 3, 14, tzinfo=timezone.utc)
-
-        # test start == end
-        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
-        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
-        # test start == end == boundary, and min_length == 0
-        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
-        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
-        # test start and end on different boundaries
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, None),
-            [floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, None),
-            [floor_day, floor_day + DAY],
-        )
-        # test min_length
-        self.assertEqual(
-            time_range(floor_hour, floor_hour + HOUR, CountStat.HOUR, 4),
-            [floor_hour - 2 * HOUR, floor_hour - HOUR, floor_hour, floor_hour + HOUR],
-        )
-        self.assertEqual(
-            time_range(floor_day, floor_day + DAY, CountStat.DAY, 4),
-            [floor_day - 2 * DAY, floor_day - DAY, floor_day, floor_day + DAY],
-        )
-
-
-class TestMapArrays(ZulipTestCase):
-    def test_map_arrays(self) -> None:
-        a = {
-            "desktop app 1.0": [1, 2, 3],
-            "desktop app 2.0": [10, 12, 13],
-            "desktop app 3.0": [21, 22, 23],
-            "website": [1, 2, 3],
-            "ZulipiOS": [1, 2, 3],
-            "ZulipElectron": [2, 5, 7],
-            "ZulipMobile": [1, 2, 3],
-            "ZulipMobile/flutter": [1, 1, 1],
-            "ZulipFlutter": [1, 1, 1],
-            "ZulipPython": [1, 2, 3],
-            "API: Python": [1, 2, 3],
-            "SomethingRandom": [4, 5, 6],
-            "ZulipGitHubWebhook": [7, 7, 9],
-            "ZulipAndroid": [64, 63, 65],
-            "ZulipTerminal": [9, 10, 11],
-        }
-        result = rewrite_client_arrays(a)
-        self.assertEqual(
-            result,
-            {
-                "Old desktop app": [32, 36, 39],
-                "Old iOS app": [1, 2, 3],
-                "Desktop app": [2, 5, 7],
-                "Mobile app (React Native)": [1, 2, 3],
-                "Mobile app beta (Flutter)": [2, 2, 2],
-                "Web app": [1, 2, 3],
-                "Python API": [2, 4, 6],
-                "SomethingRandom": [4, 5, 6],
-                "GitHub webhook": [7, 7, 9],
-                "Old Android app": [64, 63, 65],
-                "Terminal app": [9, 10, 11],
-            },
-        )

analytics/tests/test_views.py

@@ -0,0 +1,62 @@
+from django.utils.timezone import get_fixed_timezone
+from zerver.lib.test_classes import ZulipTestCase
+
+from analytics.lib.counts import CountStat
+from analytics.lib.time_utils import time_range
+from analytics.views import rewrite_client_arrays
+
+from datetime import datetime, timedelta
+
+class TestTimeRange(ZulipTestCase):
+    def test_time_range(self):
+        # type: () -> None
+        HOUR = timedelta(hours=1)
+        DAY = timedelta(days=1)
+        TZINFO = get_fixed_timezone(-100) # 100 minutes west of UTC
+
+        # Using 22:59 so that converting to UTC and applying floor_to_{hour,day} do not commute
+        a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=TZINFO)
+        floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=TZINFO)
+        floor_day = datetime(2016, 3, 14).replace(tzinfo=TZINFO)
+
+        # test start == end
+        self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
+        self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
+        # test start == end == boundary, and min_length == 0
+        self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
+        self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
+        # test start and end on different boundaries
+        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, None),
+                         [floor_hour, floor_hour+HOUR])
+        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, None),
+                         [floor_day, floor_day+DAY])
+        # test min_length
+        self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, 4),
+                         [floor_hour-2*HOUR, floor_hour-HOUR, floor_hour, floor_hour+HOUR])
+        self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, 4),
+                         [floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
+
+class TestMapArrays(ZulipTestCase):
+    def test_map_arrays(self):
+        # type: () -> None
+        a = {'desktop app 1.0': [1, 2, 3],
+             'desktop app 2.0': [10, 12, 13],
+             'desktop app 3.0': [21, 22, 23],
+             'website': [1, 2, 3],
+             'ZulipiOS': [1, 2, 3],
+             'ZulipMobile': [1, 5, 7],
+             'ZulipPython': [1, 2, 3],
+             'API: Python': [1, 2, 3],
+             'SomethingRandom': [4, 5, 6],
+             'ZulipGitHubWebhook': [7, 7, 9],
+             'ZulipAndroid': [64, 63, 65]}
+        result = rewrite_client_arrays(a)
+        self.assertEqual(result,
+                         {'Old desktop app': [32, 36, 39],
+                          'Old iOS app': [1, 2, 3],
+                          'New iOS app': [1, 5, 7],
+                          'Website': [1, 2, 3],
+                          'Python API': [2, 4, 6],
+                          'SomethingRandom': [4, 5, 6],
+                          'GitHub webhook': [7, 7, 9],
+                          'Android app': [64, 63, 65]})

analytics/urls.py

@@ -1,38 +1,22 @@
-from django.conf import settings
+from django.conf.urls import url, include
-from django.conf.urls import include
+from zerver.lib.rest import rest_dispatch
-from django.urls import path
-from django.urls.resolvers import URLPattern, URLResolver
 
-from analytics.views.stats import (
+import analytics.views
-    get_chart_data,
-    get_chart_data_for_installation,
-    get_chart_data_for_realm,
-    get_chart_data_for_stream,
-    stats,
-    stats_for_installation,
-    stats_for_realm,
-)
-from zerver.lib.rest import rest_path
 
-i18n_urlpatterns: list[URLPattern | URLResolver] = [
+i18n_urlpatterns = [
     # Server admin (user_profile.is_staff) visible stats pages
-    path("stats/realm/<realm_str>/", stats_for_realm),
+    url(r'^activity$', analytics.views.get_activity,
-    path("stats/installation", stats_for_installation),
+        name='analytics.views.get_activity'),
+    url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
+        name='analytics.views.get_realm_activity'),
+    url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
+        name='analytics.views.get_user_activity'),
+
     # User-visible stats page
-    path("stats", stats, name="stats"),
+    url(r'^stats$', analytics.views.stats,
+        name='analytics.views.stats'),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import stats_for_remote_installation, stats_for_remote_realm
-
-    i18n_urlpatterns += [
-        path("stats/remote/<int:remote_server_id>/installation", stats_for_remote_installation),
-        path(
-            "stats/remote/<int:remote_server_id>/realm/<int:remote_realm_id>/",
-            stats_for_remote_realm,
-        ),
-    ]
-
 # These endpoints are a part of the API (V1), which uses:
 # * REST verbs
 # * Basic auth (username:password is email:apiKey)
@@ -43,32 +27,13 @@ if settings.ZILENCER_ENABLED:
 # All of these paths are accessed by either a /json or /api prefix
 v1_api_and_json_patterns = [
     # get data for the graphs at /stats
-    rest_path("analytics/chart_data", GET=get_chart_data),
+    url(r'^analytics/chart_data$', rest_dispatch,
-    rest_path("analytics/chart_data/stream/<stream_id>", GET=get_chart_data_for_stream),
+        {'GET': 'analytics.views.get_chart_data'}),
-    rest_path("analytics/chart_data/realm/<realm_str>", GET=get_chart_data_for_realm),
-    rest_path("analytics/chart_data/installation", GET=get_chart_data_for_installation),
 ]
 
-if settings.ZILENCER_ENABLED:
-    from analytics.views.stats import (
-        get_chart_data_for_remote_installation,
-        get_chart_data_for_remote_realm,
-    )
-
-    v1_api_and_json_patterns += [
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/installation",
-            GET=get_chart_data_for_remote_installation,
-        ),
-        rest_path(
-            "analytics/chart_data/remote/<int:remote_server_id>/realm/<int:remote_realm_id>",
-            GET=get_chart_data_for_remote_realm,
-        ),
-    ]
-
 i18n_urlpatterns += [
-    path("api/v1/", include(v1_api_and_json_patterns)),
+    url(r'^api/v1/', include(v1_api_and_json_patterns)),
-    path("json/", include(v1_api_and_json_patterns)),
+    url(r'^json/', include(v1_api_and_json_patterns)),
 ]
 
 urlpatterns = i18n_urlpatterns

analytics/views/stats.py

@@ -1,575 +0,0 @@
-import logging
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-from typing import Any, Optional, TypeAlias, TypeVar, cast
-
-from django.conf import settings
-from django.db.models import QuerySet
-from django.http import HttpRequest, HttpResponse, HttpResponseNotFound
-from django.shortcuts import render
-from django.utils import translation
-from django.utils.timezone import now as timezone_now
-from django.utils.translation import gettext as _
-
-from analytics.lib.counts import COUNT_STATS, CountStat
-from analytics.lib.time_utils import time_range
-from analytics.models import (
-    BaseCount,
-    InstallationCount,
-    RealmCount,
-    StreamCount,
-    UserCount,
-    installation_epoch,
-)
-from zerver.decorator import (
-    require_non_guest_user,
-    require_server_admin,
-    require_server_admin_api,
-    to_utc_datetime,
-    zulip_login_required,
-)
-from zerver.lib.exceptions import JsonableError
-from zerver.lib.i18n import get_and_set_request_language, get_language_translation_data
-from zerver.lib.request import REQ, has_request_variables
-from zerver.lib.response import json_success
-from zerver.lib.streams import access_stream_by_id
-from zerver.lib.timestamp import convert_to_UTC
-from zerver.lib.validator import to_non_negative_int
-from zerver.models import Client, Realm, Stream, UserProfile
-from zerver.models.realms import get_realm
-
-if settings.ZILENCER_ENABLED:
-    from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer
-
-MAX_TIME_FOR_FULL_ANALYTICS_GENERATION = timedelta(days=1, minutes=30)
-
-
-def is_analytics_ready(realm: Realm) -> bool:
-    return (timezone_now() - realm.date_created) > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION
-
-
-def render_stats(
-    request: HttpRequest,
-    data_url_suffix: str,
-    realm: Realm | None,
-    *,
-    title: str | None = None,
-    analytics_ready: bool = True,
-) -> HttpResponse:
-    assert request.user.is_authenticated
-
-    if realm is not None:
-        # Same query to get guest user count as in get_seat_count in corporate/lib/stripe.py.
-        guest_users = UserProfile.objects.filter(
-            realm=realm, is_active=True, is_bot=False, role=UserProfile.ROLE_GUEST
-        ).count()
-        space_used = realm.currently_used_upload_space_bytes()
-        if title:
-            pass
-        else:
-            title = realm.name or realm.string_id
-    else:
-        assert title
-        guest_users = None
-        space_used = None
-
-    request_language = get_and_set_request_language(
-        request,
-        request.user.default_language,
-        translation.get_language_from_path(request.path_info),
-    )
-
-    # Sync this with stats_params_schema in base_page_params.ts.
-    page_params = dict(
-        page_type="stats",
-        data_url_suffix=data_url_suffix,
-        upload_space_used=space_used,
-        guest_users=guest_users,
-        translation_data=get_language_translation_data(request_language),
-    )
-
-    return render(
-        request,
-        "analytics/stats.html",
-        context=dict(
-            target_name=title,
-            page_params=page_params,
-            analytics_ready=analytics_ready,
-        ),
-    )
-
-
-@zulip_login_required
-def stats(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    realm = request.user.realm
-    if request.user.is_guest:
-        # TODO: Make @zulip_login_required pass the UserProfile so we
-        # can use @require_member_or_admin
-        raise JsonableError(_("Not allowed for guest users"))
-    return render_stats(request, "", realm, analytics_ready=is_analytics_ready(realm))
-
-
-@require_server_admin
-@has_request_variables
-def stats_for_realm(request: HttpRequest, realm_str: str) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        return HttpResponseNotFound()
-
-    return render_stats(
-        request,
-        f"/realm/{realm_str}",
-        realm,
-        analytics_ready=is_analytics_ready(realm),
-    )
-
-
-@require_server_admin
-@has_request_variables
-def stats_for_remote_realm(
-    request: HttpRequest, remote_server_id: int, remote_realm_id: int
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/realm/{remote_realm_id}",
-        None,
-        title=f"Realm {remote_realm_id} on server {server.hostname}",
-    )
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_realm(
-    request: HttpRequest, /, user_profile: UserProfile, realm_str: str, **kwargs: Any
-) -> HttpResponse:
-    try:
-        realm = get_realm(realm_str)
-    except Realm.DoesNotExist:
-        raise JsonableError(_("Invalid organization"))
-
-    return get_chart_data(request, user_profile, realm=realm, **kwargs)
-
-
-@require_non_guest_user
-@has_request_variables
-def get_chart_data_for_stream(
-    request: HttpRequest, /, user_profile: UserProfile, stream_id: int
-) -> HttpResponse:
-    stream, ignored_sub = access_stream_by_id(
-        user_profile,
-        stream_id,
-        require_active=True,
-        allow_realm_admin=True,
-    )
-
-    return get_chart_data(request, user_profile, stream=stream)
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_remote_realm(
-    request: HttpRequest,
-    /,
-    user_profile: UserProfile,
-    remote_server_id: int,
-    remote_realm_id: int,
-    **kwargs: Any,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return get_chart_data(
-        request,
-        user_profile,
-        server=server,
-        remote=True,
-        remote_realm_id=int(remote_realm_id),
-        **kwargs,
-    )
-
-
-@require_server_admin
-def stats_for_installation(request: HttpRequest) -> HttpResponse:
-    assert request.user.is_authenticated
-    return render_stats(request, "/installation", None, title="installation")
-
-
-@require_server_admin
-def stats_for_remote_installation(request: HttpRequest, remote_server_id: int) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return render_stats(
-        request,
-        f"/remote/{server.id}/installation",
-        None,
-        title=f"remote installation {server.hostname}",
-    )
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_installation(
-    request: HttpRequest, /, user_profile: UserProfile, chart_name: str = REQ(), **kwargs: Any
-) -> HttpResponse:
-    return get_chart_data(request, user_profile, for_installation=True, **kwargs)
-
-
-@require_server_admin_api
-@has_request_variables
-def get_chart_data_for_remote_installation(
-    request: HttpRequest,
-    /,
-    user_profile: UserProfile,
-    remote_server_id: int,
-    chart_name: str = REQ(),
-    **kwargs: Any,
-) -> HttpResponse:
-    assert settings.ZILENCER_ENABLED
-    server = RemoteZulipServer.objects.get(id=remote_server_id)
-    return get_chart_data(
-        request,
-        user_profile,
-        for_installation=True,
-        remote=True,
-        server=server,
-        **kwargs,
-    )
-
-
-@require_non_guest_user
-@has_request_variables
-def get_chart_data(
-    request: HttpRequest,
-    user_profile: UserProfile,
-    chart_name: str = REQ(),
-    min_length: int | None = REQ(converter=to_non_negative_int, default=None),
-    start: datetime | None = REQ(converter=to_utc_datetime, default=None),
-    end: datetime | None = REQ(converter=to_utc_datetime, default=None),
-    # These last several parameters are only used by functions
-    # wrapping get_chart_data; the callers are responsible for
-    # parsing/validation/authorization for them.
-    realm: Realm | None = None,
-    for_installation: bool = False,
-    remote: bool = False,
-    remote_realm_id: int | None = None,
-    server: Optional["RemoteZulipServer"] = None,
-    stream: Stream | None = None,
-) -> HttpResponse:
-    TableType: TypeAlias = (
-        type["RemoteInstallationCount"]
-        | type[InstallationCount]
-        | type["RemoteRealmCount"]
-        | type[RealmCount]
-    )
-    if for_installation:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table: TableType = RemoteInstallationCount
-            assert server is not None
-        else:
-            aggregate_table = InstallationCount
-    else:
-        if remote:
-            assert settings.ZILENCER_ENABLED
-            aggregate_table = RemoteRealmCount
-            assert server is not None
-            assert remote_realm_id is not None
-        else:
-            aggregate_table = RealmCount
-
-    tables: (
-        tuple[TableType] | tuple[TableType, type[UserCount]] | tuple[TableType, type[StreamCount]]
-    )
-
-    if chart_name == "number_of_humans":
-        stats = [
-            COUNT_STATS["1day_actives::day"],
-            COUNT_STATS["realm_active_humans::day"],
-            COUNT_STATS["active_users_audit:is_bot:day"],
-        ]
-        tables = (aggregate_table,)
-        subgroup_to_label: dict[CountStat, dict[str | None, str]] = {
-            stats[0]: {None: "_1day"},
-            stats[1]: {None: "_15day"},
-            stats[2]: {"false": "all_time"},
-        }
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_over_time":
-        stats = [COUNT_STATS["messages_sent:is_bot:hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_message_type":
-        stats = [COUNT_STATS["messages_sent:message_type:day"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {
-            stats[0]: {
-                "public_stream": _("Public channels"),
-                "private_stream": _("Private channels"),
-                "private_message": _("Direct messages"),
-                "huddle_message": _("Group direct messages"),
-            }
-        }
-        labels_sort_function = lambda data: sort_by_totals(data["everyone"])
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_client":
-        stats = [COUNT_STATS["messages_sent:client:day"]]
-        tables = (aggregate_table, UserCount)
-        # Note that the labels are further re-written by client_label_map
-        subgroup_to_label = {
-            stats[0]: {str(id): name for id, name in Client.objects.values_list("id", "name")}
-        }
-        labels_sort_function = sort_client_labels
-        include_empty_subgroups = False
-    elif chart_name == "messages_read_over_time":
-        stats = [COUNT_STATS["messages_read::hour"]]
-        tables = (aggregate_table, UserCount)
-        subgroup_to_label = {stats[0]: {None: "read"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    elif chart_name == "messages_sent_by_stream":
-        if stream is None:
-            raise JsonableError(
-                _("Missing channel for chart: {chart_name}").format(chart_name=chart_name)
-            )
-        stats = [COUNT_STATS["messages_in_stream:is_bot:day"]]
-        tables = (aggregate_table, StreamCount)
-        subgroup_to_label = {stats[0]: {"false": "human", "true": "bot"}}
-        labels_sort_function = None
-        include_empty_subgroups = True
-    else:
-        raise JsonableError(_("Unknown chart name: {chart_name}").format(chart_name=chart_name))
-
-    # Most likely someone using our API endpoint. The /stats page does not
-    # pass a start or end in its requests.
-    if start is not None:
-        start = convert_to_UTC(start)
-    if end is not None:
-        end = convert_to_UTC(end)
-    if start is not None and end is not None and start > end:
-        raise JsonableError(
-            _("Start time is later than end time. Start: {start}, End: {end}").format(
-                start=start,
-                end=end,
-            )
-        )
-
-    if realm is None:
-        # Note that this value is invalid for Remote tables; be
-        # careful not to access it in those code paths.
-        realm = user_profile.realm
-
-    if remote:
-        # For remote servers, we don't have fillstate data, and thus
-        # should simply use the first and last data points for the
-        # table.
-        assert server is not None
-        assert aggregate_table is RemoteInstallationCount or aggregate_table is RemoteRealmCount
-        aggregate_table_remote = cast(
-            type[RemoteInstallationCount] | type[RemoteRealmCount], aggregate_table
-        )  # https://stackoverflow.com/questions/68540528/mypy-assertions-on-the-types-of-types
-        if not aggregate_table_remote.objects.filter(server=server).exists():
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-        if start is None:
-            first = (
-                aggregate_table_remote.objects.filter(server=server).order_by("remote_id").first()
-            )
-            assert first is not None
-            start = first.end_time
-        if end is None:
-            last = aggregate_table_remote.objects.filter(server=server).order_by("remote_id").last()
-            assert last is not None
-            end = last.end_time
-    else:
-        # Otherwise, we can use tables on the current server to
-        # determine a nice range, and some additional validation.
-        if start is None:
-            if for_installation:
-                start = installation_epoch()
-            else:
-                start = realm.date_created
-        if end is None:
-            end = max(
-                stat.last_successful_fill() or datetime.min.replace(tzinfo=timezone.utc)
-                for stat in stats
-            )
-
-        if start > end and (timezone_now() - start > MAX_TIME_FOR_FULL_ANALYTICS_GENERATION):
-            logging.warning(
-                "User from realm %s attempted to access /stats, but the computed "
-                "start time: %s (creation of realm or installation) is later than the computed "
-                "end time: %s (last successful analytics update). Is the "
-                "analytics cron job running?",
-                realm.string_id,
-                start,
-                end,
-            )
-            raise JsonableError(
-                _("No analytics data available. Please contact your server administrator.")
-            )
-
-    assert len({stat.frequency for stat in stats}) == 1
-    end_times = time_range(start, end, stats[0].frequency, min_length)
-    data: dict[str, Any] = {
-        "end_times": [int(end_time.timestamp()) for end_time in end_times],
-        "frequency": stats[0].frequency,
-    }
-
-    aggregation_level = {
-        InstallationCount: "everyone",
-        RealmCount: "everyone",
-        UserCount: "user",
-        StreamCount: "everyone",
-    }
-    if settings.ZILENCER_ENABLED:
-        aggregation_level[RemoteInstallationCount] = "everyone"
-        aggregation_level[RemoteRealmCount] = "everyone"
-
-    # -1 is a placeholder value, since there is no relevant filtering on InstallationCount
-    id_value = {
-        InstallationCount: -1,
-        RealmCount: realm.id,
-        UserCount: user_profile.id,
-    }
-    if stream is not None:
-        id_value[StreamCount] = stream.id
-
-    if settings.ZILENCER_ENABLED:
-        if server is not None:
-            id_value[RemoteInstallationCount] = server.id
-        # TODO: RemoteRealmCount logic doesn't correctly handle
-        # filtering by server_id as well.
-        if remote_realm_id is not None:
-            id_value[RemoteRealmCount] = remote_realm_id
-
-    for table in tables:
-        data[aggregation_level[table]] = {}
-        for stat in stats:
-            data[aggregation_level[table]].update(
-                get_time_series_by_subgroup(
-                    stat,
-                    table,
-                    id_value[table],
-                    end_times,
-                    subgroup_to_label[stat],
-                    include_empty_subgroups,
-                )
-            )
-
-    if labels_sort_function is not None:
-        data["display_order"] = labels_sort_function(data)
-    else:
-        data["display_order"] = None
-    return json_success(request, data=data)
-
-
-def sort_by_totals(value_arrays: dict[str, list[int]]) -> list[str]:
-    totals = sorted(((sum(values), label) for label, values in value_arrays.items()), reverse=True)
-    return [label for total, label in totals]
-
-
-# For any given user, we want to show a fixed set of clients in the chart,
-# regardless of the time aggregation or whether we're looking at realm or
-# user data. This fixed set ideally includes the clients most important in
-# understanding the realm's traffic and the user's traffic. This function
-# tries to rank the clients so that taking the first N elements of the
-# sorted list has a reasonable chance of doing so.
-def sort_client_labels(data: dict[str, dict[str, list[int]]]) -> list[str]:
-    realm_order = sort_by_totals(data["everyone"])
-    user_order = sort_by_totals(data["user"])
-    label_sort_values: dict[str, float] = {label: i for i, label in enumerate(realm_order)}
-    for i, label in enumerate(user_order):
-        label_sort_values[label] = min(i - 0.1, label_sort_values.get(label, i))
-    return [label for label, sort_value in sorted(label_sort_values.items(), key=lambda x: x[1])]
-
-
-CountT = TypeVar("CountT", bound=BaseCount)
-
-
-def table_filtered_to_id(table: type[CountT], key_id: int) -> QuerySet[CountT]:
-    if table == RealmCount:
-        return table._default_manager.filter(realm_id=key_id)
-    elif table == UserCount:
-        return table._default_manager.filter(user_id=key_id)
-    elif table == StreamCount:
-        return table._default_manager.filter(stream_id=key_id)
-    elif table == InstallationCount:
-        return table._default_manager.all()
-    elif settings.ZILENCER_ENABLED and table == RemoteInstallationCount:
-        return table._default_manager.filter(server_id=key_id)
-    elif settings.ZILENCER_ENABLED and table == RemoteRealmCount:
-        return table._default_manager.filter(realm_id=key_id)
-    else:
-        raise AssertionError(f"Unknown table: {table}")
-
-
-def client_label_map(name: str) -> str:
-    if name == "website":
-        return "Web app"
-    if name.startswith("desktop app"):
-        return "Old desktop app"
-    if name == "ZulipElectron":
-        return "Desktop app"
-    if name == "ZulipTerminal":
-        return "Terminal app"
-    if name == "ZulipAndroid":
-        return "Old Android app"
-    if name == "ZulipiOS":
-        return "Old iOS app"
-    if name == "ZulipMobile":
-        return "Mobile app (React Native)"
-    if name in ["ZulipFlutter", "ZulipMobile/flutter"]:
-        return "Mobile app beta (Flutter)"
-    if name in ["ZulipPython", "API: Python"]:
-        return "Python API"
-    if name.startswith("Zulip") and name.endswith("Webhook"):
-        return name[len("Zulip") : -len("Webhook")] + " webhook"
-    return name
-
-
-def rewrite_client_arrays(value_arrays: dict[str, list[int]]) -> dict[str, list[int]]:
-    mapped_arrays: dict[str, list[int]] = {}
-    for label, array in value_arrays.items():
-        mapped_label = client_label_map(label)
-        if mapped_label in mapped_arrays:
-            for i in range(len(array)):
-                mapped_arrays[mapped_label][i] += value_arrays[label][i]
-        else:
-            mapped_arrays[mapped_label] = [value_arrays[label][i] for i in range(len(array))]
-    return mapped_arrays
-
-
-def get_time_series_by_subgroup(
-    stat: CountStat,
-    table: type[BaseCount],
-    key_id: int,
-    end_times: list[datetime],
-    subgroup_to_label: dict[str | None, str],
-    include_empty_subgroups: bool,
-) -> dict[str, list[int]]:
-    queryset = (
-        table_filtered_to_id(table, key_id)
-        .filter(property=stat.property)
-        .values_list("subgroup", "end_time", "value")
-    )
-    value_dicts: dict[str | None, dict[datetime, int]] = defaultdict(lambda: defaultdict(int))
-    for subgroup, end_time, value in queryset:
-        value_dicts[subgroup][end_time] = value
-    value_arrays = {}
-    for subgroup, label in subgroup_to_label.items():
-        if (subgroup in value_dicts) or include_empty_subgroups:
-            value_arrays[label] = [value_dicts[subgroup][end_time] for end_time in end_times]
-
-    if stat == COUNT_STATS["messages_sent:client:day"]:
-        # HACK: We rewrite these arrays to collapse the Client objects
-        # with similar names into a single sum, and generally give
-        # them better names
-        return rewrite_client_arrays(value_arrays)
-    return value_arrays

api/MANIFEST.in

@@ -0,0 +1,11 @@
+recursive-include integrations *
+include README.md
+include examples/zuliprc
+include examples/send-message
+include examples/subscribe
+include examples/get-public-streams
+include examples/unsubscribe
+include examples/list-members
+include examples/list-subscriptions
+include examples/print-messages
+include examples/recent-messages

api/README.md

@@ -0,0 +1,172 @@
+#### Dependencies
+
+The [Zulip API](https://zulipchat.com/api) Python bindings require the
+following Python libraries:
+
+* requests (version >= 0.12.1)
+* simplejson
+* six
+* typing (version >= 3.5.2.2)
+
+#### Installing
+
+This package uses distutils, so you can just run:
+
+    python setup.py install
+
+#### Using the API
+
+For now, the only fully supported API operation is sending a message.
+The other API queries work, but are under active development, so
+please make sure we know you're using them so that we can notify you
+as we make any changes to them.
+
+The easiest way to use these API bindings is to base your tools off
+of the example tools under examples/ in this distribution.
+
+If you place your API key in the config file `~/.zuliprc` the Python
+API bindings will automatically read it in. The format of the config
+file is as follows:
+
+    [api]
+    key=<api key from the web interface>
+    email=<your email address>
+    site=<your Zulip server's URI>
+    insecure=<true or false, true means do not verify the server certificate>
+    cert_bundle=<path to a file containing CA or server certificates to trust>
+
+If omitted, these settings have the following defaults:
+
+    insecure=false
+    cert_bundle=<the default CA bundle trusted by Python>
+
+Alternatively, you may explicitly use "--user", "--api-key", and
+`--site` in our examples, which is especially useful when testing.  If
+you are running several bots which share a home directory, we
+recommend using `--config` to specify the path to the `zuliprc` file
+for a specific bot.
+
+The command line equivalents for other configuration options are:
+
+    --insecure
+    --cert-bundle=<file>
+
+You can obtain your Zulip API key, create bots, and manage bots all
+from your Zulip settings page; with current Zulip there's also a
+button to download a `zuliprc` file for your account/server pair.
+
+A typical simple bot sending API messages will look as follows:
+
+At the top of the file:
+
+    # Make sure the Zulip API distribution's root directory is in sys.path, then:
+    import zulip
+    zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
+
+When you want to send a message:
+
+    message = {
+      "type": "stream",
+      "to": ["support"],
+      "subject": "your subject",
+      "content": "your content",
+    }
+    zulip_client.send_message(message)
+
+If you are parsing arguments, you may find it useful to use Zulip's
+option group; see any of our API examples for details on how to do this.
+
+Additional examples:
+
+    client.send_message({'type': 'stream', 'content': 'Zulip rules!',
+                         'subject': 'feedback', 'to': ['support']})
+    client.send_message({'type': 'private', 'content': 'Zulip rules!',
+                         'to': ['user1@example.com', 'user2@example.com']})
+
+send_message() returns a dict guaranteed to contain the following
+keys: msg, result.  For successful calls, result will be "success" and
+msg will be the empty string.  On error, result will be "error" and
+msg will describe what went wrong.
+
+#### Examples
+
+The API bindings package comes with several nice example scripts that
+show how to use the APIs; they are installed as part of the API
+bindings bundle.
+
+#### Logging
+
+The Zulip API comes with a ZulipStream class which can be used with the
+logging module:
+
+```
+import zulip
+import logging
+stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
+logger = logging.getLogger("your_logger")
+logger.addHandler(logging.StreamHandler(stream))
+logger.setLevel(logging.DEBUG)
+logger.info("This is an INFO test.")
+logger.debug("This is a DEBUG test.")
+logger.warn("This is a WARN test.")
+logger.error("This is a ERROR test.")
+```
+
+#### Sending messages
+
+You can use the included `zulip-send` script to send messages via the
+API directly from existing scripts.
+
+    zulip-send hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+Alternatively, if you don't want to use your ~/.zuliprc file:
+
+    zulip-send --user shakespeare-bot@example.com \
+        --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
+        --site https://zulip.example.com \
+        hamlet@example.com cordelia@example.com -m \
+        "Conscience doth make cowards of us all."
+
+#### Working with an untrusted server certificate
+
+If your server has either a self-signed certificate, or a certificate signed
+by a CA that you don't wish to globally trust then by default the API will
+fail with an SSL verification error.
+
+You can add `insecure=true` to your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    insecure=true
+
+This disables verification of the server certificate, so connections are
+encrypted but unauthenticated. This is not secure, but may be good enough
+for a development environment.
+
+
+You can explicitly trust the server certificate using `cert_bundle=<filename>`
+in your .zuliprc file.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/zulip.example.com.crt
+
+You can also explicitly trust a different set of Certificate Authorities from
+the default bundle that is trusted by Python. For example to trust a company
+internal CA.
+
+    [api]
+    site=https://zulip.example.com
+    cert_bundle=/home/bots/certs/example.com.ca-bundle
+
+Save the server certificate (or the CA certificate) in its own file,
+converting to PEM format first if necessary.
+Verify that the certificate you have saved is the same as the one on the
+server.
+
+The `cert_bundle` option trusts the server / CA certificate only for
+interaction with the zulip site, and is relatively secure.
+
+Note that a certificate bundle is merely one or more certificates combined
+into a single file.

api/bin/zulip-send

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# zulip-send -- Sends a message to the specified recipients.
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import sys
+import os
+import optparse
+import logging
+
+from typing import Any, Dict, List, Optional
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+import zulip
+
+logging.basicConfig()
+
+log = logging.getLogger('zulip-send')
+
+def do_send_message(client, message_data):
+    # type: (zulip.Client, Dict[str, Any]) -> bool
+    '''Sends a message and optionally prints status about the same.'''
+
+    if message_data['type'] == 'stream':
+        log.info('Sending message to stream "%s", subject "%s"... ' %
+                 (message_data['to'], message_data['subject']))
+    else:
+        log.info('Sending message to %s... ' % message_data['to'])
+    response = client.send_message(message_data)
+    if response['result'] == 'success':
+        log.info('Message sent.')
+        return True
+    else:
+        log.error(response['msg'])
+        return False
+
+def main(argv=None):
+    # type: (Optional[List[str]]) -> int
+    if argv is None:
+        argv = sys.argv
+
+    usage = """%prog [options] [recipient...]
+
+    Sends a message specified recipients.
+
+    Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
+              %prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
+
+    These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
+    '--user' and '--api-key' arguments.
+    """
+
+    parser = optparse.OptionParser(usage=usage)
+
+    # Grab parser options from the API common set
+    parser.add_option_group(zulip.generate_option_group(parser))
+
+    parser.add_option('-m', '--message',
+                      help='Specifies the message to send, prevents interactive prompting.')
+
+    group = optparse.OptionGroup(parser, 'Stream parameters')
+    group.add_option('-s', '--stream',
+                     dest='stream',
+                     action='store',
+                     help='Allows the user to specify a stream for the message.')
+    group.add_option('-S', '--subject',
+                     dest='subject',
+                     action='store',
+                     help='Allows the user to specify a subject for the message.')
+    parser.add_option_group(group)
+
+    (options, recipients) = parser.parse_args(argv[1:])
+
+    if options.verbose:
+        logging.getLogger().setLevel(logging.INFO)
+    # Sanity check user data
+    if len(recipients) != 0 and (options.stream or options.subject):
+        parser.error('You cannot specify both a username and a stream/subject.')
+    if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
+        parser.error('Stream messages must have a subject')
+    if len(recipients) == 0 and not (options.stream and options.subject):
+        parser.error('You must specify a stream/subject or at least one recipient.')
+
+    client = zulip.init_from_options(options)
+
+    if not options.message:
+        options.message = sys.stdin.read()
+
+    if options.stream:
+        message_data = {
+            'type': 'stream',
+            'content': options.message,
+            'subject': options.subject,
+            'to': options.stream,
+        }
+    else:
+        message_data = {
+            'type': 'private',
+            'content': options.message,
+            'to': recipients,
+        }
+
+    if not do_send_message(client, message_data):
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

api/examples/create-user

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+from os import path
+import optparse
+
+usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
+
+Create a user. You must be a realm admin to use this API, and the user
+will be created in your realm.
+
+Example: create-user --site=http://localhost:9991 --user=rwbarton@example.com --new-email=jarthur@example.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
+"""
+
+sys.path.append(path.join(path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--new-email')
+parser.add_option('--new-password')
+parser.add_option('--new-full-name')
+parser.add_option('--new-short-name')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.create_user({
+    'email': options.new_email,
+    'password': options.new_password,
+    'full_name': options.new_full_name,
+    'short_name': options.new_short_name
+}))

api/examples/edit-message

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
+
+Edits a message that you sent
+
+Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--message-id', default="")
+parser.add_option('--subject', default="")
+parser.add_option('--content', default="")
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "message_id": options.message_id,
+}
+if options.subject != "":
+    message_data["subject"] = options.subject
+if options.content != "":
+    message_data["content"] = options.content
+print(client.update_message(message_data))

api/examples/get-public-streams

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out all the public streams in the realm.
+
+Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.get_streams(include_public=True, include_subscribed=False))

api/examples/list-members

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
+
+List the names and e-mail addresses of the people in your realm.
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+for user in client.get_members()["members"]:
+    print(user["full_name"], user["email"])

api/examples/list-subscriptions

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out a list of the user's subscriptions.
+
+Example: list-subscriptions --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+print(client.list_subscriptions())

api/examples/print-events

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-events --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out certain events received by the indicated bot or user matching the filter below.
+
+Example: print-events --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_event(event):
+    # type: (Dict[str, Any]) -> None
+    print(event)
+
+# This is a blocking call, and will continuously poll for new events
+# Note also the filter here is messages to the stream Denmark; if you
+# don't specify event_types it'll print all events.
+client.call_on_each_event(print_event, event_types=["message"], narrow=[["stream", "Denmark"]])

api/examples/print-messages

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+from typing import Any, Dict
+
+usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
+
+Prints out each message received by the indicated bot or user.
+
+Example: print-messages --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+def print_message(message):
+    # type: (Dict[str, Any]) -> None
+    print(message)
+
+# This is a blocking call, and will continuously poll for new messages
+client.call_on_each_message(print_message)

api/examples/recent-messages

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import json
+import optparse
+
+usage = """recent-messages [options] --count=<no. of previous messages> --user=<sender's email address> --api-key=<sender's api key>
+
+Prints out last count messages received by the indicated bot or user
+
+Example: recent-messages --count=101 --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--count', default=100)
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+request = {
+    'narrow': [["stream", "Denmark"]],
+    'num_before': options.count,
+    'num_after': 0,
+    'anchor': 1000000000,
+    'apply_markdown': False
+}
+
+old_messages = client.call_endpoint(
+    url='messages',
+    method='GET',
+    request=request,
+)
+
+if 'messages' in old_messages:
+    for message in old_messages['messages']:
+        print(json.dumps(message, indent=4))
+else:
+    print([])

api/examples/send-message

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
+
+Sends a test message to the specified recipients.
+
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
+Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--subject', default="test")
+parser.add_option('--message', default="test message")
+parser.add_option('--type', default='private')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+if len(args) == 0:
+    parser.error("You must specify recipients")
+
+client = zulip.init_from_options(options)
+
+message_data = {
+    "type": options.type,
+    "content": options.message,
+    "subject": options.subject,
+    "to": args,
+}
+print(client.send_message(message_data))

api/examples/subscribe

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is subscribed to the listed streams.
+
+Examples: subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          subscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.add_subscriptions([{"name": stream_name} for stream_name in
+                                options.streams.split()]))

api/examples/unsubscribe

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import sys
+import os
+import optparse
+
+usage = """unsubscribe  --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
+
+Ensures the user is not subscribed to the listed streams.
+
+Examples: unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
+          unsubscribe --user=username@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+"""
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+import zulip
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option_group(zulip.generate_option_group(parser))
+parser.add_option('--streams', default='')
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+if options.streams == "":
+    print("Usage:", parser.usage, file=sys.stderr)
+    sys.exit(1)
+
+print(client.remove_subscriptions(options.streams.split()))

api/examples/upload-file

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright © 2012-2017 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from __future__ import print_function
+import importlib
+import optparse
+import sys
+
+from six.moves import StringIO as _StringIO
+sys.path.insert(0, './api')
+from typing import IO
+import zulip
+
+class StringIO(_StringIO):
+    name = '' # https://github.com/python/typeshed/issues/598
+
+usage = """upload-file --user=<user's email address> --api-key=<user's api key> [options]
+
+Upload a file, and print the corresponding URI.
+
+Example: upload-file --user=cordelia@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --file-path=cat.png
+
+You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
+If no --file-path is specified, a placeholder text file will be used instead.
+"""
+
+parser = optparse.OptionParser(usage=usage)
+parser.add_option('--file-path')
+parser.add_option_group(zulip.generate_option_group(parser))
+(options, args) = parser.parse_args()
+
+client = zulip.init_from_options(options)
+
+file = None # type: IO
+if options.file_path:
+    file = open(options.file_path, 'rb')
+else:
+    file = StringIO('This is a test file.')
+    file.name = 'test.txt'
+
+response = client.upload_file(file)
+
+try:
+    print('File URI: {}'.format(response['uri']))
+except KeyError:
+    print('Error! API response was: {}'.format(response))

api/examples/zuliprc

@@ -0,0 +1,4 @@
+; Save this file as ~/.zuliprc
+[api]
+key=<your bot's api key from the web interface>
+email=<your bot's email address>

api/integrations/asana/zulip_asana_config.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+### REQUIRED CONFIGURATION ###
+
+# Change these values to your Asana credentials.
+ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# Change these values to the credentials for your Asana bot.
+ZULIP_USER = "asana-bot@example.com"
+ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
+
+# The Zulip stream that will receive Asana task updates.
+ZULIP_STREAM_NAME = "asana"
+
+
+### OPTIONAL CONFIGURATION ###
+
+# Set to None for logging to stdout when testing, and to a file for
+# logging in production.
+#LOG_FILE = "/var/tmp/zulip_asana.log"
+LOG_FILE = None
+
+# This file is used to resume this mirror in case the script shuts down.
+# It is required and needs to be writeable.
+RESUME_FILE = "/var/tmp/zulip_asana.state"
+
+# When initially started, how many hours of messages to include.
+ASANA_INITIAL_HISTORY_HOURS = 1
+
+# Set this to your Zulip API server URI
+ZULIP_SITE = "https://zulip.example.com"
+
+# If properly installed, the Zulip API should be in your import
+# path, but if not, set a custom path below
+ZULIP_API_PATH = None

api/integrations/asana/zulip_asana_mirror

@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Asana integration for Zulip
+#
+# Copyright © 2014 Zulip, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# The "zulip_asana_mirror" script is run continuously, possibly on a work computer
+# or preferably on a server.
+#
+# When restarted, it will attempt to pick up where it left off.
+#
+# python-dateutil is a dependency for this script.
+
+from __future__ import print_function
+import base64
+from datetime import datetime, timedelta
+from typing import List, Dict, Optional, Any, Tuple
+
+import json
+import logging
+import os
+import time
+from six.moves import urllib
+from six.moves.urllib import request as urllib_request
+import sys
+
+
+try:
+    import dateutil.parser
+    from dateutil.tz import gettz
+except ImportError as e:
+    print(e, file=sys.stderr)
+    print("Please install the python-dateutil package.", file=sys.stderr)
+    exit(1)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import zulip_asana_config as config
+VERSION = "0.9"
+
+if config.ZULIP_API_PATH is not None:
+    sys.path.append(config.ZULIP_API_PATH)
+import zulip
+
+if config.LOG_FILE:
+    logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
+else:
+    logging.basicConfig(level=logging.INFO)
+
+client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
+                      site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
+
+def fetch_from_asana(path):
+    # type: (str) -> Optional[Dict[str, Any]]
+    """
+    Request a resource through the Asana API, authenticating using
+    HTTP basic auth.
+    """
+    auth = base64.encodestring(b'%s:' % (config.ASANA_API_KEY,))
+    headers = {"Authorization": "Basic %s" % auth}
+
+    url = "https://app.asana.com/api/1.0" + path
+    request = urllib_request.Request(url, None, headers) # type: ignore
+    result = urllib_request.urlopen(request) # type: ignore
+
+    return json.load(result)
+
+def send_zulip(topic, content):
+    # type: (str, str) -> Dict[str, str]
+    """
+    Send a message to Zulip using the configured stream and bot credentials.
+    """
+    message = {"type": "stream",
+               "sender": config.ZULIP_USER,
+               "to": config.ZULIP_STREAM_NAME,
+               "subject": topic,
+               "content": content,
+               }
+    return client.send_message(message)
+
+def datestring_to_datetime(datestring):
+    # type: (str) -> datetime
+    """
+    Given an ISO 8601 datestring, return the corresponding datetime object.
+    """
+    return dateutil.parser.parse(datestring).replace(
+        tzinfo=gettz('Z'))
+
+class TaskDict(dict):
+    """
+    A helper class to turn a dictionary with task information into an
+    object where each of the keys is an attribute for easy access.
+    """
+    def __getattr__(self, field):
+        # type: (TaskDict, str) -> Any
+        return self.get(field)
+
+def format_topic(task, projects):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string that will be the Zulip message topic for this task.
+    """
+    # Tasks can be associated with multiple projects, but in practice they seem
+    # to mostly be associated with one.
+    project_name = projects[task.projects[0]["id"]]
+    return "%s: %s" % (project_name, task.name)
+
+def format_assignee(task, users):
+    # type: (TaskDict, Dict[str, str]) -> str
+    """
+    Return a string describing the task's assignee.
+    """
+    if task.assignee:
+        assignee_name = users[task.assignee["id"]]
+        assignee_info = "**Assigned to**: %s (%s)" % (
+            assignee_name, task.assignee_status)
+    else:
+        assignee_info = "**Status**: Unassigned"
+
+    return assignee_info
+
+def format_due_date(task):
+    # type: (TaskDict) -> str
+    """
+    Return a string describing the task's due date.
+    """
+    if task.due_on:
+        due_date_info = "**Due on**: %s" % (task.due_on,)
+    else:
+        due_date_info = "**Due date**: None"
+    return due_date_info
+
+def format_task_creation_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a newly-created task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** created:
+
+~~~ quote
+%s
+~~~
+
+%s
+%s
+""" % (task.name, task.notes, assignee_info, due_date_info)
+    return topic, content
+
+def format_task_completion_event(task, projects, users):
+    # type: (TaskDict, Dict[str, str], Dict[str, str]) -> Tuple[str, str]
+    """
+    Format the topic and content for a completed task.
+    """
+    topic = format_topic(task, projects)
+    assignee_info = format_assignee(task, users)
+    due_date_info = format_due_date(task)
+
+    content = """Task **%s** completed. :white_check_mark:
+
+%s
+%s
+""" % (task.name, assignee_info, due_date_info)
+    return topic, content
+
+def since():
+    # type: () -> datetime
+    """
+    Return a newness threshold for task events to be processed.
+    """
+    # If we have a record of the last event processed and it is recent, use it,
+    # else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
+    def default_since():
+        # type: () -> datetime
+        return datetime.utcnow() - timedelta(
+            hours=config.ASANA_INITIAL_HISTORY_HOURS)
+
+    if os.path.exists(config.RESUME_FILE):
+        try:
+            with open(config.RESUME_FILE, "r") as f:
+                datestring = f.readline().strip()
+                timestamp = float(datestring)
+                max_timestamp_processed = datetime.fromtimestamp(timestamp)
+                logging.info("Reading from resume file: " + datestring)
+        except (ValueError, IOError) as e:
+            logging.warn("Could not open resume file: " + str(e))
+            max_timestamp_processed = default_since()
+    else:
+        logging.info("No resume file, processing an initial history.")
+        max_timestamp_processed = default_since()
+
+    # Even if we can read a timestamp from RESUME_FILE, if it is old don't use
+    # it.
+    return max(max_timestamp_processed, default_since())
+
+def process_new_events():
+    # type: () -> None
+    """
+    Forward new Asana task events to Zulip.
+    """
+    # In task queries, Asana only exposes IDs for projects and users, so we need
+    # to look up the mappings.
+    projects = dict((elt["id"], elt["name"]) for elt in
+                    fetch_from_asana("/projects")["data"])
+    users = dict((elt["id"], elt["name"]) for elt in
+                 fetch_from_asana("/users")["data"])
+
+    cutoff = since()
+    max_timestamp_processed = cutoff
+    time_operations = (("created_at", format_task_creation_event),
+                       ("completed_at", format_task_completion_event))
+    task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
+                   "modified_at", "due_on", "name", "notes", "projects"]
+
+    # First, gather all of the tasks that need processing. We'll
+    # process them in order.
+    new_events = []
+
+    for project_id in projects:
+        project_url = "/projects/%d/tasks?opt_fields=%s" % (
+            project_id, ",".join(task_fields))
+        tasks = fetch_from_asana(project_url)["data"]
+
+        for task in tasks:
+            task = TaskDict(task)
+
+            for time_field, operation in time_operations:
+                if task[time_field]:
+                    operation_time = datestring_to_datetime(task[time_field])
+                    if operation_time > cutoff:
+                        new_events.append((operation_time, time_field, operation, task))
+
+    new_events.sort()
+    now = datetime.utcnow()
+
+    for operation_time, time_field, operation, task in new_events:
+        # Unfortunately, creating an Asana task is not an atomic operation. If
+        # the task was just created, or is missing basic information, it is
+        # probably because the task is still being filled out -- wait until the
+        # next round to process it.
+        if (time_field == "created_at") and \
+                (now - operation_time < timedelta(seconds=30)):
+            # The task was just created, give the user some time to fill out
+            # more information.
+            return
+
+        if (time_field == "created_at") and (not task.name) and \
+                (now - operation_time < timedelta(seconds=60)):
+            # If this new task hasn't had a name for a full 30 seconds, assume
+            # you don't plan on giving it one.
+            return
+
+        topic, content = operation(task, projects, users)
+        logging.info("Sending Zulip for " + topic)
+        result = send_zulip(topic, content)
+
+        # If the Zulip wasn't sent successfully, don't update the
+        # max timestamp processed so the task has another change to
+        # be forwarded. Exit, giving temporary issues time to
+        # resolve.
+        if not result.get("result"):
+            logging.warn("Malformed result, exiting:")
+            logging.warn(str(result))
+            return
+
+        if result["result"] != "success":
+            logging.warn(result["msg"])
+            return
+
+        if operation_time > max_timestamp_processed:
+            max_timestamp_processed = operation_time
+
+    if max_timestamp_processed > cutoff:
+        max_datestring = max_timestamp_processed.strftime("%s.%f")
+        logging.info("Updating resume file: " + max_datestring)
+        open(config.RESUME_FILE, 'w').write(max_datestring)
+
+while True:
+    try:
+        process_new_events()
+        time.sleep(5)
+    except KeyboardInterrupt:
+        logging.info("Shutting down...")
+        logging.info("Set LOG_FILE to log to a file instead of stdout.")
+        break