mirror of
https://github.com/zulip/zulip.git
synced 2025-10-23 04:52:12 +00:00
Compare commits
1 Commits
2.1.8
...
enterprise
| Author | SHA1 | Date | |
|---|---|---|---|
|
2a67775c27 |
@@ -1,6 +0,0 @@
|
||||
> 0.2%
|
||||
> 0.2% in US
|
||||
last 2 versions
|
||||
Firefox ESR
|
||||
not dead
|
||||
Chrome 26 # similar to PhantomJS
|
||||
@@ -1,151 +0,0 @@
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
|
||||
# high-level documentation on our CircleCI setup.
|
||||
# See CircleCI upstream's docs on this config format:
|
||||
# https://circleci.com/docs/2.0/language-python/
|
||||
#
|
||||
version: 2.0
|
||||
aliases:
|
||||
- &create_cache_directories
|
||||
run:
|
||||
name: create cache directories
|
||||
command: |
|
||||
dirs=(/srv/zulip-{npm,venv}-cache)
|
||||
sudo mkdir -p "${dirs[@]}"
|
||||
sudo chown -R circleci "${dirs[@]}"
|
||||
|
||||
- &restore_cache_package_json
|
||||
restore_cache:
|
||||
keys:
|
||||
- v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
|
||||
|
||||
- &restore_cache_requirements
|
||||
restore_cache:
|
||||
keys:
|
||||
- v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
|
||||
|
||||
- &install_dependencies
|
||||
run:
|
||||
name: install dependencies
|
||||
command: |
|
||||
sudo apt-get update
|
||||
# Install moreutils so we can use `ts` and `mispipe` in the following.
|
||||
sudo apt-get install -y moreutils
|
||||
|
||||
# CircleCI sets the following in Git config at clone time:
|
||||
# url.ssh://git@github.com.insteadOf https://github.com
|
||||
# This breaks the Git clones in the NVM `install.sh` we run
|
||||
# in `install-node`.
|
||||
# TODO: figure out why that breaks, and whether we want it.
|
||||
# (Is it an optimization?)
|
||||
rm -f /home/circleci/.gitconfig
|
||||
|
||||
# This is the main setup job for the test suite
|
||||
mispipe "tools/ci/setup-backend" ts
|
||||
|
||||
# Cleaning caches is mostly unnecessary in Circle, because
|
||||
# most builds don't get to write to the cache.
|
||||
# mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
|
||||
|
||||
- &save_cache_package_json
|
||||
save_cache:
|
||||
paths:
|
||||
- /srv/zulip-npm-cache
|
||||
key: v1-npm-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "package.json" }}-{{ checksum "yarn.lock" }}
|
||||
|
||||
- &save_cache_requirements
|
||||
save_cache:
|
||||
paths:
|
||||
- /srv/zulip-venv-cache
|
||||
key: v1-venv-base.{{ .Environment.CIRCLE_JOB }}-{{ checksum "requirements/thumbor.txt" }}-{{ checksum "requirements/dev.txt" }}
|
||||
# TODO: in Travis we also cache ~/zulip-emoji-cache, ~/node, ~/misc
|
||||
|
||||
- &run_backend_tests
|
||||
run:
|
||||
name: run backend tests
|
||||
command: |
|
||||
. /srv/zulip-py3-venv/bin/activate
|
||||
mispipe ./tools/ci/backend ts
|
||||
|
||||
- &run_frontend_tests
|
||||
run:
|
||||
name: run frontend tests
|
||||
command: |
|
||||
. /srv/zulip-py3-venv/bin/activate
|
||||
mispipe ./tools/ci/frontend ts
|
||||
|
||||
- &upload_coverage_report
|
||||
run:
|
||||
name: upload coverage report
|
||||
command: |
|
||||
. /srv/zulip-py3-venv/bin/activate
|
||||
pip install codecov && codecov \
|
||||
|| echo "Error in uploading coverage reports to codecov.io."
|
||||
|
||||
jobs:
|
||||
"xenial-backend-frontend-python3.5":
|
||||
docker:
|
||||
# This is built from tools/circleci/images/xenial/Dockerfile .
|
||||
# Xenial ships with Python 3.5.
|
||||
- image: gregprice/circleci:xenial-python-4.test
|
||||
|
||||
working_directory: ~/zulip
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- *create_cache_directories
|
||||
- *restore_cache_package_json
|
||||
- *restore_cache_requirements
|
||||
- *install_dependencies
|
||||
- *save_cache_package_json
|
||||
- *save_cache_requirements
|
||||
- *run_backend_tests
|
||||
- *run_frontend_tests
|
||||
# We only need to upload coverage reports on whichever platform
|
||||
# runs the frontend tests.
|
||||
- *upload_coverage_report
|
||||
|
||||
- store_artifacts:
|
||||
path: ./var/casper/
|
||||
destination: casper
|
||||
|
||||
- store_artifacts:
|
||||
path: ../../../tmp/zulip-test-event-log/
|
||||
destination: test-reports
|
||||
|
||||
- store_test_results:
|
||||
path: ./var/xunit-test-results/casper/
|
||||
|
||||
"bionic-backend-python3.6":
|
||||
docker:
|
||||
# This is built from tools/circleci/images/bionic/Dockerfile .
|
||||
# Bionic ships with Python 3.6.
|
||||
- image: gregprice/circleci:bionic-python-1.test
|
||||
|
||||
working_directory: ~/zulip
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
|
||||
- *create_cache_directories
|
||||
|
||||
- run:
|
||||
name: do Bionic hack
|
||||
command: |
|
||||
# Temporary hack till `sudo service redis-server start` gets fixes in Bionic. See
|
||||
# https://chat.zulip.org/#narrow/stream/3-backend/topic/Ubuntu.20bionic.20CircleCI
|
||||
sudo sed -i '/^bind/s/bind.*/bind 0.0.0.0/' /etc/redis/redis.conf
|
||||
|
||||
- *restore_cache_package_json
|
||||
- *restore_cache_requirements
|
||||
- *install_dependencies
|
||||
- *save_cache_package_json
|
||||
- *save_cache_requirements
|
||||
- *run_backend_tests
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- "xenial-backend-frontend-python3.5"
|
||||
- "bionic-backend-python3.6"
|
||||
12
.codecov.yml
12
.codecov.yml
@@ -1,12 +0,0 @@
|
||||
comment: off
|
||||
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: auto
|
||||
# Codecov has the tendency to report a lot of false negatives,
|
||||
# so we basically suppress comments completely.
|
||||
threshold: 50%
|
||||
base: auto
|
||||
patch: off
|
||||
@@ -1,25 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.{sh,py,pyi,js,ts,json,yml,xml,css,md,markdown,handlebars,html}]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
[*.py]
|
||||
max_line_length = 110
|
||||
|
||||
[*.{js,ts}]
|
||||
max_line_length = 100
|
||||
|
||||
[*.{svg,rb,pp,pl}]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
||||
[*.cfg]
|
||||
indent_style = space
|
||||
indent_size = 8
|
||||
@@ -1,10 +0,0 @@
|
||||
# This is intended for generated files and vendored third-party files.
|
||||
# For our source code, instead of adding files here, consider using
|
||||
# specific eslint-disable comments in the files themselves.
|
||||
|
||||
/docs/_build
|
||||
/static/generated
|
||||
/static/third
|
||||
/static/webpack-bundles
|
||||
/var
|
||||
/zulip-py3-venv
|
||||
496
.eslintrc.json
496
.eslintrc.json
@@ -1,496 +0,0 @@
|
||||
{
|
||||
"env": {
|
||||
"node": true,
|
||||
"es6": true
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2019,
|
||||
"warnOnUnsupportedTypeScriptVersion": false,
|
||||
"sourceType": "module"
|
||||
},
|
||||
"globals": {
|
||||
"$": false,
|
||||
"ClipboardJS": false,
|
||||
"Dict": false,
|
||||
"FetchStatus": false,
|
||||
"Filter": false,
|
||||
"Handlebars": false,
|
||||
"LightboxCanvas": false,
|
||||
"MessageListData": false,
|
||||
"MessageListView": false,
|
||||
"Plotly": false,
|
||||
"SockJS": false,
|
||||
"Socket": false,
|
||||
"Sortable": false,
|
||||
"WinChan": false,
|
||||
"XDate": false,
|
||||
"_": false,
|
||||
"activity": false,
|
||||
"admin": false,
|
||||
"alert_words": false,
|
||||
"alert_words_ui": false,
|
||||
"attachments_ui": false,
|
||||
"avatar": false,
|
||||
"billing": false,
|
||||
"blueslip": false,
|
||||
"bot_data": false,
|
||||
"bridge": false,
|
||||
"buddy_data": false,
|
||||
"buddy_list": false,
|
||||
"channel": false,
|
||||
"click_handlers": false,
|
||||
"color_data": false,
|
||||
"colorspace": false,
|
||||
"common": false,
|
||||
"components": false,
|
||||
"compose": false,
|
||||
"compose_actions": false,
|
||||
"compose_fade": false,
|
||||
"compose_pm_pill": false,
|
||||
"compose_state": false,
|
||||
"compose_ui": false,
|
||||
"composebox_typeahead": false,
|
||||
"condense": false,
|
||||
"confirm_dialog": false,
|
||||
"copy_and_paste": false,
|
||||
"csrf_token": false,
|
||||
"current_msg_list": true,
|
||||
"drafts": false,
|
||||
"echo": false,
|
||||
"emoji": false,
|
||||
"emoji_codes": false,
|
||||
"emoji_picker": false,
|
||||
"favicon": false,
|
||||
"feature_flags": false,
|
||||
"feedback_widget": false,
|
||||
"fenced_code": false,
|
||||
"flatpickr": false,
|
||||
"floating_recipient_bar": false,
|
||||
"gear_menu": false,
|
||||
"hash_util": false,
|
||||
"hashchange": false,
|
||||
"helpers": false,
|
||||
"history": false,
|
||||
"home_msg_list": false,
|
||||
"hotspots": false,
|
||||
"i18n": false,
|
||||
"info_overlay": false,
|
||||
"input_pill": false,
|
||||
"invite": false,
|
||||
"jQuery": false,
|
||||
"katex": false,
|
||||
"keydown_util": false,
|
||||
"lightbox": false,
|
||||
"list_cursor": false,
|
||||
"list_render": false,
|
||||
"list_util": false,
|
||||
"loading": false,
|
||||
"localStorage": false,
|
||||
"local_message": false,
|
||||
"localstorage": false,
|
||||
"location": false,
|
||||
"markdown": false,
|
||||
"marked": false,
|
||||
"md5": false,
|
||||
"message_edit": false,
|
||||
"message_events": false,
|
||||
"message_fetch": false,
|
||||
"message_flags": false,
|
||||
"message_list": false,
|
||||
"message_live_update": false,
|
||||
"message_scroll": false,
|
||||
"message_store": false,
|
||||
"message_util": false,
|
||||
"message_viewport": false,
|
||||
"moment": false,
|
||||
"muting": false,
|
||||
"muting_ui": false,
|
||||
"narrow": false,
|
||||
"narrow_state": false,
|
||||
"navigate": false,
|
||||
"night_mode": false,
|
||||
"notifications": false,
|
||||
"overlays": false,
|
||||
"padded_widget": false,
|
||||
"page_params": false,
|
||||
"panels": false,
|
||||
"people": false,
|
||||
"pm_conversations": false,
|
||||
"pm_list": false,
|
||||
"pointer": false,
|
||||
"popovers": false,
|
||||
"presence": false,
|
||||
"pygments_data": false,
|
||||
"reactions": false,
|
||||
"realm_icon": false,
|
||||
"realm_logo": false,
|
||||
"realm_night_logo": false,
|
||||
"recent_senders": false,
|
||||
"reload": false,
|
||||
"reload_state": false,
|
||||
"reminder": false,
|
||||
"resize": false,
|
||||
"rows": false,
|
||||
"rtl": false,
|
||||
"run_test": false,
|
||||
"schema": false,
|
||||
"scroll_bar": false,
|
||||
"scroll_util": false,
|
||||
"search": false,
|
||||
"search_pill": false,
|
||||
"search_pill_widget": false,
|
||||
"search_suggestion": false,
|
||||
"search_util": false,
|
||||
"sent_messages": false,
|
||||
"server_events": false,
|
||||
"server_events_dispatch": false,
|
||||
"settings": false,
|
||||
"settings_account": false,
|
||||
"settings_bots": false,
|
||||
"settings_display": false,
|
||||
"settings_emoji": false,
|
||||
"settings_exports": false,
|
||||
"settings_linkifiers": false,
|
||||
"settings_invites": false,
|
||||
"settings_muting": false,
|
||||
"settings_notifications": false,
|
||||
"settings_org": false,
|
||||
"settings_panel_menu": false,
|
||||
"settings_profile_fields": false,
|
||||
"settings_sections": false,
|
||||
"settings_streams": false,
|
||||
"settings_toggle": false,
|
||||
"settings_ui": false,
|
||||
"settings_user_groups": false,
|
||||
"settings_users": false,
|
||||
"starred_messages": false,
|
||||
"stream_color": false,
|
||||
"stream_create": false,
|
||||
"stream_data": false,
|
||||
"stream_edit": false,
|
||||
"stream_events": false,
|
||||
"stream_list": false,
|
||||
"stream_muting": false,
|
||||
"stream_popover": false,
|
||||
"stream_sort": false,
|
||||
"stream_ui_updates": false,
|
||||
"StripeCheckout": false,
|
||||
"submessage": false,
|
||||
"subs": false,
|
||||
"tab_bar": false,
|
||||
"templates": false,
|
||||
"tictactoe_widget": false,
|
||||
"timerender": false,
|
||||
"todo_widget": false,
|
||||
"top_left_corner": false,
|
||||
"topic_data": false,
|
||||
"topic_generator": false,
|
||||
"topic_list": false,
|
||||
"topic_zoom": false,
|
||||
"transmit": false,
|
||||
"tutorial": false,
|
||||
"typeahead_helper": false,
|
||||
"typing": false,
|
||||
"typing_data": false,
|
||||
"typing_events": false,
|
||||
"ui": false,
|
||||
"ui_init": false,
|
||||
"ui_report": false,
|
||||
"ui_util": false,
|
||||
"unread": false,
|
||||
"unread_ops": false,
|
||||
"unread_ui": false,
|
||||
"upgrade": false,
|
||||
"upload": false,
|
||||
"upload_widget": false,
|
||||
"user_events": false,
|
||||
"user_groups": false,
|
||||
"user_pill": false,
|
||||
"user_search": false,
|
||||
"user_status": false,
|
||||
"user_status_ui": false,
|
||||
"util": false,
|
||||
"poll_widget": false,
|
||||
"widgetize": false,
|
||||
"zcommand": false,
|
||||
"zform": false,
|
||||
"zxcvbn": false
|
||||
},
|
||||
"plugins": [
|
||||
"eslint-plugin-empty-returns"
|
||||
],
|
||||
"rules": {
|
||||
"array-callback-return": "error",
|
||||
"array-bracket-spacing": "error",
|
||||
"arrow-spacing": [ "error", { "before": true, "after": true } ],
|
||||
"block-scoped-var": "error",
|
||||
"brace-style": [ "error", "1tbs", { "allowSingleLine": true } ],
|
||||
"camelcase": "off",
|
||||
"comma-dangle": [ "error",
|
||||
{
|
||||
"arrays": "always-multiline",
|
||||
"objects": "always-multiline",
|
||||
"imports": "always-multiline",
|
||||
"exports": "always-multiline",
|
||||
"functions": "never"
|
||||
}
|
||||
],
|
||||
"comma-spacing": [ "error",
|
||||
{
|
||||
"before": false,
|
||||
"after": true
|
||||
}
|
||||
],
|
||||
"complexity": [ "off", 4 ],
|
||||
"curly": "error",
|
||||
"dot-notation": [ "error", { "allowKeywords": true } ],
|
||||
"empty-returns/main": "error",
|
||||
"eol-last": [ "error", "always" ],
|
||||
"eqeqeq": "error",
|
||||
"func-style": [ "off", "expression" ],
|
||||
"guard-for-in": "error",
|
||||
"indent": ["error", 4, {
|
||||
"ArrayExpression": "first",
|
||||
"ObjectExpression": "first",
|
||||
"SwitchCase": 0,
|
||||
"CallExpression": {"arguments": "first"},
|
||||
"FunctionExpression": {"parameters": "first"},
|
||||
"FunctionDeclaration": {"parameters": "first"}
|
||||
}],
|
||||
"key-spacing": [ "error",
|
||||
{
|
||||
"beforeColon": false,
|
||||
"afterColon": true
|
||||
}
|
||||
],
|
||||
"keyword-spacing": [ "error",
|
||||
{
|
||||
"before": true,
|
||||
"after": true,
|
||||
"overrides": {
|
||||
"return": { "after": true },
|
||||
"throw": { "after": true },
|
||||
"case": { "after": true }
|
||||
}
|
||||
}
|
||||
],
|
||||
"max-depth": [ "off", 4 ],
|
||||
"max-len": [ "error", 100, 2,
|
||||
{
|
||||
"ignoreUrls": true,
|
||||
"ignoreComments": false,
|
||||
"ignoreRegExpLiterals": true,
|
||||
"ignoreStrings": true,
|
||||
"ignoreTemplateLiterals": true
|
||||
}
|
||||
],
|
||||
"max-params": [ "off", 3 ],
|
||||
"max-statements": [ "off", 10 ],
|
||||
"new-cap": [ "error",
|
||||
{
|
||||
"newIsCap": true,
|
||||
"capIsNew": false
|
||||
}
|
||||
],
|
||||
"new-parens": "error",
|
||||
"newline-per-chained-call": "off",
|
||||
"no-alert": "error",
|
||||
"no-array-constructor": "error",
|
||||
"no-bitwise": "error",
|
||||
"no-caller": "error",
|
||||
"no-case-declarations": "error",
|
||||
"no-catch-shadow": "error",
|
||||
"no-console": "off",
|
||||
"no-const-assign": "error",
|
||||
"no-control-regex": "error",
|
||||
"no-debugger": "error",
|
||||
"no-delete-var": "error",
|
||||
"no-div-regex": "error",
|
||||
"no-dupe-class-members": "error",
|
||||
"no-dupe-keys": "error",
|
||||
"no-duplicate-imports": "error",
|
||||
"no-else-return": "error",
|
||||
"no-empty": "error",
|
||||
"no-empty-character-class": "error",
|
||||
"no-eq-null": "error",
|
||||
"no-eval": "error",
|
||||
"no-ex-assign": "error",
|
||||
"no-extra-parens": ["error", "all"],
|
||||
"no-extra-semi": "error",
|
||||
"no-fallthrough": "error",
|
||||
"no-floating-decimal": "error",
|
||||
"no-func-assign": "error",
|
||||
"no-implied-eval": "error",
|
||||
"no-iterator": "error",
|
||||
"no-label-var": "error",
|
||||
"no-labels": "error",
|
||||
"no-loop-func": "error",
|
||||
"no-mixed-requires": [ "off", false ],
|
||||
"no-multi-str": "error",
|
||||
"no-native-reassign": "error",
|
||||
"no-nested-ternary": "off",
|
||||
"no-new-func": "error",
|
||||
"no-new-object": "error",
|
||||
"no-new-wrappers": "error",
|
||||
"no-obj-calls": "error",
|
||||
"no-octal": "error",
|
||||
"no-octal-escape": "error",
|
||||
"no-param-reassign": "off",
|
||||
"no-plusplus": "error",
|
||||
"no-proto": "error",
|
||||
"no-redeclare": "error",
|
||||
"no-regex-spaces": "error",
|
||||
"no-restricted-syntax": "off",
|
||||
"no-return-assign": "error",
|
||||
"no-script-url": "error",
|
||||
"no-self-compare": "error",
|
||||
"no-shadow": "off",
|
||||
"no-sync": "error",
|
||||
"no-ternary": "off",
|
||||
"no-trailing-spaces": "error",
|
||||
"no-undef": "error",
|
||||
"no-undef-init": "error",
|
||||
"no-underscore-dangle": "off",
|
||||
"no-unneeded-ternary": [ "error", { "defaultAssignment": false } ],
|
||||
"no-unreachable": "error",
|
||||
"no-unused-expressions": "error",
|
||||
"no-unused-vars": [ "error",
|
||||
{
|
||||
"vars": "local",
|
||||
"args": "after-used",
|
||||
"varsIgnorePattern": "print_elapsed_time|check_duplicate_ids"
|
||||
}
|
||||
],
|
||||
"no-use-before-define": "error",
|
||||
"no-useless-constructor": "error",
|
||||
// The Zulip codebase complies partially with the "no-useless-escape"
|
||||
// rule; only regex expressions haven't been updated yet.
|
||||
// Updated regex expressions are currently being tested in casper
|
||||
// files and will decide about a potential future enforcement of this rule.
|
||||
"no-useless-escape": "off",
|
||||
"no-var": "error",
|
||||
"space-unary-ops": "error",
|
||||
"no-whitespace-before-property": "error",
|
||||
"no-with": "error",
|
||||
"one-var": [ "error", "never" ],
|
||||
"padded-blocks": "off",
|
||||
"prefer-const": [ "error",
|
||||
{
|
||||
"destructuring": "any",
|
||||
"ignoreReadBeforeAssign": true
|
||||
}
|
||||
],
|
||||
"quote-props": [ "error", "as-needed",
|
||||
{
|
||||
"keywords": false,
|
||||
"unnecessary": true,
|
||||
"numbers": false
|
||||
}
|
||||
],
|
||||
"quotes": [ "off", "single" ],
|
||||
"radix": "error",
|
||||
"semi": "error",
|
||||
"semi-spacing": ["error", {"before": false, "after": true}],
|
||||
"sort-imports": "error",
|
||||
"space-before-blocks": "error",
|
||||
"space-before-function-paren": [ "error",
|
||||
{
|
||||
"anonymous": "always",
|
||||
"named": "never",
|
||||
"asyncArrow": "always"
|
||||
}
|
||||
],
|
||||
"space-in-parens": "error",
|
||||
"space-infix-ops": "error",
|
||||
"spaced-comment": "off",
|
||||
"strict": "off",
|
||||
"template-curly-spacing": "error",
|
||||
"unnecessary-strict": "off",
|
||||
"use-isnan": "error",
|
||||
"valid-typeof": [ "error", { "requireStringLiterals": true } ],
|
||||
"wrap-iife": [ "error", "outside", { "functionPrototypeMethods": false } ],
|
||||
"wrap-regex": "off",
|
||||
"yoda": "error"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"files": [
|
||||
"frontend_tests/casper_tests/*.js",
|
||||
"frontend_tests/casper_lib/*.js"
|
||||
],
|
||||
"rules": {
|
||||
"no-var": "off" // PhantomJS doesn’t support let, const
|
||||
}
|
||||
},
|
||||
{
|
||||
"files": ["**/*.ts"],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"project": "tsconfig.json"
|
||||
},
|
||||
"plugins": ["@typescript-eslint"],
|
||||
"rules": {
|
||||
// Disable base rule to avoid conflict
|
||||
"empty-returns/main": "off",
|
||||
"indent": "off",
|
||||
"func-call-spacing": "off",
|
||||
"no-magic-numbers": "off",
|
||||
"semi": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-useless-constructor": "off",
|
||||
|
||||
"@typescript-eslint/adjacent-overload-signatures": "error",
|
||||
"@typescript-eslint/array-type": "error",
|
||||
"@typescript-eslint/await-thenable": "error",
|
||||
"@typescript-eslint/ban-types": "error",
|
||||
"@typescript-eslint/ban-ts-ignore": "off",
|
||||
"@typescript-eslint/camelcase": "off",
|
||||
"@typescript-eslint/class-name-casing": "error",
|
||||
"@typescript-eslint/consistent-type-assertions": "error",
|
||||
"@typescript-eslint/explicit-function-return-type": ["error", { "allowExpressions": true }],
|
||||
"@typescript-eslint/explicit-member-accessibility": "off",
|
||||
"@typescript-eslint/func-call-spacing": "error",
|
||||
"@typescript-eslint/generic-type-naming": "off",
|
||||
"@typescript-eslint/indent": "error",
|
||||
"@typescript-eslint/interface-name-prefix": "off",
|
||||
"@typescript-eslint/member-delimiter-style": "error",
|
||||
"@typescript-eslint/member-naming": ["error", { "private": "^_" } ],
|
||||
"@typescript-eslint/member-ordering": "error",
|
||||
"@typescript-eslint/no-array-constructor": "error",
|
||||
"@typescript-eslint/no-empty-interface": "error",
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-extraneous-class": "error",
|
||||
"@typescript-eslint/no-for-in-array": "off",
|
||||
"@typescript-eslint/no-inferrable-types": "error",
|
||||
"@typescript-eslint/no-magic-numbers": "off",
|
||||
"@typescript-eslint/no-misused-new": "error",
|
||||
"@typescript-eslint/no-namespace": "error",
|
||||
"@typescript-eslint/no-non-null-assertion": "off",
|
||||
"@typescript-eslint/no-parameter-properties": "error",
|
||||
"@typescript-eslint/no-require-imports": "off",
|
||||
"@typescript-eslint/no-this-alias": "off",
|
||||
"@typescript-eslint/no-type-alias": "off",
|
||||
"@typescript-eslint/no-unnecessary-qualifier": "error",
|
||||
"@typescript-eslint/no-unnecessary-type-assertion": "error",
|
||||
"@typescript-eslint/no-unused-vars": ["error", { "varsIgnorePattern": "^_" } ],
|
||||
"@typescript-eslint/no-use-before-define": "error",
|
||||
"@typescript-eslint/no-useless-constructor": "error",
|
||||
"@typescript-eslint/no-var-requires": "off",
|
||||
"@typescript-eslint/prefer-for-of": "off",
|
||||
"@typescript-eslint/prefer-function-type": "off",
|
||||
"@typescript-eslint/prefer-includes": "error",
|
||||
"@typescript-eslint/prefer-interface": "off",
|
||||
"@typescript-eslint/prefer-namespace-keyword": "error",
|
||||
"@typescript-eslint/prefer-regexp-exec": "error",
|
||||
"@typescript-eslint/prefer-string-starts-ends-with": "error",
|
||||
"@typescript-eslint/promise-function-async": "error",
|
||||
"@typescript-eslint/restrict-plus-operands": "off",
|
||||
"@typescript-eslint/semi": "error",
|
||||
"@typescript-eslint/triple-slash-reference": "error",
|
||||
"@typescript-eslint/type-annotation-spacing": "error",
|
||||
"@typescript-eslint/unbound-method": "off",
|
||||
"@typescript-eslint/unified-signatures": "error"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
33
.gitattributes
vendored
33
.gitattributes
vendored
@@ -1,13 +1,20 @@
|
||||
* text=auto eol=lf
|
||||
*.gif binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.eot binary
|
||||
*.woff binary
|
||||
*.woff2 binary
|
||||
*.svg binary
|
||||
*.ttf binary
|
||||
*.png binary
|
||||
*.otf binary
|
||||
*.tif binary
|
||||
*.ogg binary
|
||||
.gitignore export-ignore
|
||||
.gitattributes export-ignore
|
||||
/analytics export-ignore
|
||||
/assets export-ignore
|
||||
/bots export-ignore
|
||||
/corporate export-ignore
|
||||
/static export-ignore
|
||||
/tools export-ignore
|
||||
/zilencer export-ignore
|
||||
/templates/analytics export-ignore
|
||||
/templates/corporate export-ignore
|
||||
/templates/zilencer export-ignore
|
||||
/puppet/zulip_internal export-ignore
|
||||
/zproject/local_settings.py export-ignore
|
||||
/zproject/test_settings.py export-ignore
|
||||
/zerver/fixtures export-ignore
|
||||
/zerver/tests.py export-ignore
|
||||
/zerver/tests export-ignore
|
||||
/node_modules export-ignore
|
||||
/humbug export-ignore
|
||||
|
||||
14
.github/pull_request_template.md
vendored
14
.github/pull_request_template.md
vendored
@@ -1,14 +0,0 @@
|
||||
<!-- What's this PR for? (Just a link to an issue is fine.) -->
|
||||
|
||||
|
||||
**Testing Plan:** <!-- How have you tested? -->
|
||||
|
||||
|
||||
**GIFs or Screenshots:** <!-- If a UI change. See:
|
||||
https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html
|
||||
-->
|
||||
|
||||
|
||||
<!-- Also be sure to make clear, coherent commits:
|
||||
https://zulip.readthedocs.io/en/latest/contributing/version-control.html
|
||||
-->
|
||||
94
.gitignore
vendored
94
.gitignore
vendored
@@ -1,82 +1,28 @@
|
||||
# Quick format and style primer:
|
||||
#
|
||||
# * If a pattern is meant only for a specific location, it should have a
|
||||
# leading slash, like `/staticfiles.json`.
|
||||
# * In principle any non-trailing slash (like `zproject/dev-secrets.conf`)
|
||||
# will do, but this makes a confusing pattern. Adding a leading slash
|
||||
# is clearer.
|
||||
#
|
||||
# * Patterns like `.vscode/` without slashes, or with only a trailing slash,
|
||||
# match in any subdirectory.
|
||||
#
|
||||
# * Subdirectories with several internal things to ignore get their own
|
||||
# `.gitignore` files.
|
||||
#
|
||||
# * Comments must be on their own line. (Otherwise they don't work.)
|
||||
#
|
||||
# See `git help ignore` for details on the format.
|
||||
|
||||
## Config files for the dev environment
|
||||
/zproject/dev-secrets.conf
|
||||
/tools/conf.ini
|
||||
/tools/custom_provision
|
||||
/tools/droplets/conf.ini
|
||||
|
||||
## Byproducts of setting up and using the dev environment
|
||||
*.pyc
|
||||
package-lock.json
|
||||
|
||||
/.vagrant
|
||||
/var
|
||||
|
||||
/.dmypy.json
|
||||
|
||||
# Dockerfiles generated for CircleCI
|
||||
/tools/circleci/images
|
||||
|
||||
# Generated i18n data
|
||||
/locale/en
|
||||
/locale/language_options.json
|
||||
/locale/language_name_map.json
|
||||
/locale/*/mobile.json
|
||||
|
||||
# Static build
|
||||
*.mo
|
||||
npm-debug.log
|
||||
/node_modules
|
||||
/prod-static
|
||||
/staticfiles.json
|
||||
/webpack-stats-production.json
|
||||
/yarn-error.log
|
||||
zulip-git-version
|
||||
|
||||
# Test / analysis tools
|
||||
.coverage
|
||||
|
||||
## Files (or really symlinks) created in a prod deployment
|
||||
/zproject/prod_settings.py
|
||||
/zulip-current-venv
|
||||
/zulip-py3-venv
|
||||
|
||||
## Files left by various editors and local environments
|
||||
# (Ideally these should be in everyone's respective personal gitignore files.)
|
||||
*~
|
||||
/all_messages_log.*
|
||||
/event_log/*
|
||||
/server.log
|
||||
/update-prod-static.log
|
||||
/prod-static
|
||||
/errors/*
|
||||
*.sw[po]
|
||||
.idea
|
||||
.DS_Store
|
||||
event_queues.pickle
|
||||
stats/
|
||||
zerver/fixtures/available-migrations
|
||||
zerver/fixtures/migration-status
|
||||
zerver/fixtures/test_data1.json
|
||||
.kdev4
|
||||
zulip.kdev4
|
||||
memcached_prefix
|
||||
coverage/
|
||||
/queue_error
|
||||
/workers.log
|
||||
.test-js-with-node.html
|
||||
digest.log
|
||||
errors.log
|
||||
manage.log
|
||||
.kateproject.d/
|
||||
.kateproject
|
||||
*.kate-swp
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
.vscode/
|
||||
*.DS_Store
|
||||
# .cache/ is generated by VSCode's test runner
|
||||
.cache/
|
||||
.eslintcache
|
||||
|
||||
## Miscellaneous
|
||||
# (Ideally this section is empty.)
|
||||
zthumbor/thumbor_local_settings.py
|
||||
.transifexrc
|
||||
|
||||
13
.gitlint
13
.gitlint
@@ -1,13 +0,0 @@
|
||||
[general]
|
||||
ignore=title-trailing-punctuation, body-min-length, body-is-missing, title-imperative-mood
|
||||
|
||||
extra-path=tools/lib/gitlint-rules.py
|
||||
|
||||
[title-match-regex-allow-exception]
|
||||
regex=^(.+:\ )?[A-Z].+\.$
|
||||
|
||||
[title-max-length]
|
||||
line-length=76
|
||||
|
||||
[body-max-line-length]
|
||||
line-length=76
|
||||
10
.isort.cfg
10
.isort.cfg
@@ -1,10 +0,0 @@
|
||||
[settings]
|
||||
line_length = 79
|
||||
multi_line_output = 2
|
||||
balanced_wrapping = true
|
||||
known_third_party = django, ujson, sqlalchemy
|
||||
known_first_party = zerver, zproject, version, confirmation, zilencer, analytics, frontend_tests, scripts, corporate
|
||||
sections = FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
|
||||
lines_after_imports = 1
|
||||
# See the comment related to ioloop_logging for why this is skipped.
|
||||
skip = zerver/management/commands/runtornado.py
|
||||
67
.stylelintrc
67
.stylelintrc
@@ -1,67 +0,0 @@
|
||||
{
|
||||
"rules": {
|
||||
# Stylistic rules for CSS.
|
||||
"function-comma-space-after": "always",
|
||||
"function-comma-space-before": "never",
|
||||
"function-max-empty-lines": 0,
|
||||
"function-whitespace-after": "always",
|
||||
|
||||
"value-keyword-case": "lower",
|
||||
"value-list-comma-newline-after": "always-multi-line",
|
||||
"value-list-comma-space-after": "always-single-line",
|
||||
"value-list-comma-space-before": "never",
|
||||
"value-list-max-empty-lines": 0,
|
||||
|
||||
"unit-case": "lower",
|
||||
"property-case": "lower",
|
||||
"color-hex-case": "lower",
|
||||
|
||||
"declaration-bang-space-before": "always",
|
||||
"declaration-colon-newline-after": "always-multi-line",
|
||||
"declaration-colon-space-after": "always-single-line",
|
||||
"declaration-colon-space-before": "never",
|
||||
"declaration-block-semicolon-newline-after": "always",
|
||||
"declaration-block-semicolon-space-before": "never",
|
||||
"declaration-block-trailing-semicolon": "always",
|
||||
|
||||
"block-closing-brace-empty-line-before": "never",
|
||||
"block-closing-brace-newline-after": "always",
|
||||
"block-closing-brace-newline-before": "always",
|
||||
"block-opening-brace-newline-after": "always",
|
||||
"block-opening-brace-space-before": "always",
|
||||
|
||||
"selector-attribute-brackets-space-inside": "never",
|
||||
"selector-attribute-operator-space-after": "never",
|
||||
"selector-attribute-operator-space-before": "never",
|
||||
"selector-combinator-space-after": "always",
|
||||
"selector-combinator-space-before": "always",
|
||||
"selector-descendant-combinator-no-non-space": true,
|
||||
"selector-pseudo-class-parentheses-space-inside": "never",
|
||||
"selector-pseudo-element-case": "lower",
|
||||
"selector-pseudo-element-colon-notation": "double",
|
||||
"selector-type-case": "lower",
|
||||
"selector-list-comma-newline-after": "always",
|
||||
"selector-list-comma-space-before": "never",
|
||||
|
||||
"media-feature-colon-space-after": "always",
|
||||
"media-feature-colon-space-before": "never",
|
||||
"media-feature-name-case": "lower",
|
||||
"media-feature-parentheses-space-inside": "never",
|
||||
"media-feature-range-operator-space-after": "always",
|
||||
"media-feature-range-operator-space-before": "always",
|
||||
"media-query-list-comma-newline-after": "always",
|
||||
"media-query-list-comma-space-before": "never",
|
||||
|
||||
"at-rule-name-case": "lower",
|
||||
"at-rule-name-space-after": "always",
|
||||
"at-rule-semicolon-newline-after": "always",
|
||||
"at-rule-semicolon-space-before": "never",
|
||||
|
||||
"comment-whitespace-inside": "always",
|
||||
"indentation": 4,
|
||||
|
||||
# Limit language features
|
||||
"color-no-hex": true,
|
||||
"color-named": "never",
|
||||
}
|
||||
}
|
||||
66
.travis.yml
66
.travis.yml
@@ -1,66 +0,0 @@
|
||||
# See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
|
||||
# high-level documentation on our Travis CI setup.
|
||||
dist: xenial
|
||||
install:
|
||||
# Disable sometimes-broken sources.list in Travis base images
|
||||
- sudo rm -vf /etc/apt/sources.list.d/*
|
||||
- sudo apt-get update
|
||||
|
||||
# Disable Travis CI's built-in NVM installation
|
||||
- mispipe "mv ~/.nvm ~/.travis-nvm-disabled" ts
|
||||
|
||||
# Install codecov, the library for the code coverage reporting tool we use
|
||||
# With a retry to minimize impact of transient networking errors.
|
||||
- mispipe "pip install codecov" ts || mispipe "pip install codecov" ts
|
||||
|
||||
# This is the main setup job for the test suite
|
||||
- mispipe "tools/ci/setup-$TEST_SUITE" ts
|
||||
|
||||
# Clean any caches that are not in use to avoid our cache
|
||||
# becoming huge.
|
||||
- mispipe "scripts/lib/clean-unused-caches --verbose --threshold 0" ts
|
||||
|
||||
script:
|
||||
# We unset GEM_PATH here as a hack to work around Travis CI having
|
||||
# broken running their system puppet with Ruby. See
|
||||
# https://travis-ci.org/zulip/zulip/jobs/240120991 for an example traceback.
|
||||
- unset GEM_PATH
|
||||
- mispipe "./tools/ci/$TEST_SUITE" ts
|
||||
cache:
|
||||
yarn: true
|
||||
apt: false
|
||||
directories:
|
||||
- $HOME/zulip-venv-cache
|
||||
- $HOME/zulip-npm-cache
|
||||
- $HOME/zulip-emoji-cache
|
||||
- $HOME/node
|
||||
- $HOME/misc
|
||||
env:
|
||||
global:
|
||||
- BOTO_CONFIG=/nonexistent
|
||||
language: python
|
||||
# Our test suites generally run on Python 3.5, the version in
|
||||
# Ubuntu 16.04 xenial, which is the oldest OS release we support.
|
||||
matrix:
|
||||
include:
|
||||
# Travis will actually run the jobs in the order they're listed here;
|
||||
# that doesn't seem to be documented, but it's what we see empirically.
|
||||
# We only get 4 jobs running at a time, so we try to make the first few
|
||||
# the most likely to break.
|
||||
- python: "3.5"
|
||||
env: TEST_SUITE=production
|
||||
# Other suites moved to CircleCI -- see .circleci/.
|
||||
sudo: required
|
||||
addons:
|
||||
artifacts:
|
||||
paths:
|
||||
# Casper debugging data (screenshots, etc.) is super useful for
|
||||
# debugging test flakes.
|
||||
- $(ls var/casper/* | tr "\n" ":")
|
||||
- $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
|
||||
postgresql: "9.5"
|
||||
apt:
|
||||
packages:
|
||||
- moreutils
|
||||
after_success:
|
||||
- codecov
|
||||
33
.tx/config
33
.tx/config
@@ -1,33 +0,0 @@
|
||||
[main]
|
||||
host = https://www.transifex.com
|
||||
lang_map = zh-Hans: zh_Hans, zh-Hant: zh_Hant
|
||||
|
||||
[zulip.djangopo]
|
||||
file_filter = locale/<lang>/LC_MESSAGES/django.po
|
||||
source_file = locale/en/LC_MESSAGES/django.po
|
||||
source_lang = en
|
||||
type = PO
|
||||
|
||||
[zulip.translationsjson]
|
||||
file_filter = locale/<lang>/translations.json
|
||||
source_file = locale/en/translations.json
|
||||
source_lang = en
|
||||
type = KEYVALUEJSON
|
||||
|
||||
[zulip.mobile]
|
||||
file_filter = locale/<lang>/mobile.json
|
||||
source_file = locale/en/mobile.json
|
||||
source_lang = en
|
||||
type = KEYVALUEJSON
|
||||
|
||||
[zulip-test.djangopo]
|
||||
file_filter = locale/<lang>/LC_MESSAGES/django.po
|
||||
source_file = locale/en/LC_MESSAGES/django.po
|
||||
source_lang = en
|
||||
type = PO
|
||||
|
||||
[zulip-test.translationsjson]
|
||||
file_filter = locale/<lang>/translations.json
|
||||
source_file = locale/en/translations.json
|
||||
source_lang = en
|
||||
type = KEYVALUEJSON
|
||||
@@ -1,105 +0,0 @@
|
||||
# Zulip Code of Conduct
|
||||
|
||||
Like the technical community as a whole, the Zulip team and community is
|
||||
made up of a mixture of professionals and volunteers from all over the
|
||||
world, working on every aspect of the mission, including mentorship,
|
||||
teaching, and connecting people.
|
||||
|
||||
Diversity is one of our huge strengths, but it can also lead to
|
||||
communication issues and unhappiness. To that end, we have a few ground
|
||||
rules that we ask people to adhere to. This code applies equally to
|
||||
founders, mentors, and those seeking help and guidance.
|
||||
|
||||
This isn't an exhaustive list of things that you can't do. Rather, take it
|
||||
in the spirit in which it's intended --- a guide to make it easier to enrich
|
||||
all of us and the technical communities in which we participate.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
The following behaviors are expected and requested of all community members:
|
||||
|
||||
* Participate. In doing so, you contribute to the health and longevity of
|
||||
the community.
|
||||
* Exercise consideration and respect in your speech and actions.
|
||||
* Attempt collaboration before conflict. Assume good faith.
|
||||
* Refrain from demeaning, discriminatory, or harassing behavior and speech.
|
||||
* Take action or alert community leaders if you notice a dangerous
|
||||
situation, someone in distress, or violations of this code, even if they
|
||||
seem inconsequential.
|
||||
* Community event venues may be shared with members of the public; be
|
||||
respectful to all patrons of these locations.
|
||||
|
||||
## Unacceptable Behavior
|
||||
|
||||
The following behaviors are considered harassment and are unacceptable
|
||||
within the Zulip community:
|
||||
|
||||
* Jokes or derogatory language that singles out members of any race,
|
||||
ethnicity, culture, national origin, color, immigration status, social and
|
||||
economic class, educational level, language proficiency, sex, sexual
|
||||
orientation, gender identity and expression, age, size, family status,
|
||||
political belief, religion, and mental and physical ability.
|
||||
* Violence, threats of violence, or violent language directed against
|
||||
another person.
|
||||
* Disseminating or threatening to disseminate another person's personal
|
||||
information.
|
||||
* Personal insults of any sort.
|
||||
* Posting or displaying sexually explicit or violent material.
|
||||
* Inappropriate photography or recording.
|
||||
* Deliberate intimidation, stalking, or following (online or in person).
|
||||
* Unwelcome sexual attention. This includes sexualized comments or jokes,
|
||||
inappropriate touching or groping, and unwelcomed sexual advances.
|
||||
* Sustained disruption of community events, including talks and
|
||||
presentations.
|
||||
* Advocating for, or encouraging, any of the behaviors above.
|
||||
|
||||
## Reporting and Enforcement
|
||||
|
||||
Harassment and other code of conduct violations reduce the value of the
|
||||
community for everyone. If someone makes you or anyone else feel unsafe or
|
||||
unwelcome, please report it to the community organizers at
|
||||
zulip-code-of-conduct@googlegroups.com as soon as possible. You can make a
|
||||
report either personally or anonymously.
|
||||
|
||||
If a community member engages in unacceptable behavior, the community
|
||||
organizers may take any action they deem appropriate, up to and including a
|
||||
temporary ban or permanent expulsion from the community without warning (and
|
||||
without refund in the case of a paid event).
|
||||
|
||||
If someone outside the development community (e.g. a user of the Zulip
|
||||
software) engages in unacceptable behavior that affects someone in the
|
||||
community, we still want to know. Even if we don't have direct control over
|
||||
the violator, the community organizers can still support the people
|
||||
affected, reduce the chance of a similar violation in the future, and take
|
||||
any direct action we can.
|
||||
|
||||
The nature of reporting means it can only help after the fact. If you see
|
||||
something you can do while a violation is happening, do it. A lot of the
|
||||
harms of harassment and other violations can be mitigated by the victim
|
||||
knowing that the other people present are on their side.
|
||||
|
||||
All reports will be kept confidential. In some cases, we may determine that a
|
||||
public statement will need to be made. In such cases, the identities of all
|
||||
victims and reporters will remain confidential unless those individuals
|
||||
instruct us otherwise.
|
||||
|
||||
## Scope
|
||||
|
||||
We expect all community participants (contributors, paid or otherwise,
|
||||
sponsors, and other guests) to abide by this Code of Conduct in all
|
||||
community venues, online and in-person, as well as in all private
|
||||
communications pertaining to community business.
|
||||
|
||||
This Code of Conduct and its related procedures also applies to unacceptable
|
||||
behavior occurring outside the scope of community activities when such
|
||||
behavior has the potential to adversely affect the safety and well-being of
|
||||
community members.
|
||||
|
||||
## License and Attribution
|
||||
|
||||
This Code of Conduct is adapted from the
|
||||
[Citizen Code of Conduct](http://citizencodeofconduct.org/) and the
|
||||
[Django Code of Conduct](https://www.djangoproject.com/conduct/), and is
|
||||
under a
|
||||
[Creative Commons BY-SA](http://creativecommons.org/licenses/by-sa/4.0/)
|
||||
license.
|
||||
351
CONTRIBUTING.md
351
CONTRIBUTING.md
@@ -1,351 +0,0 @@
|
||||
# Contributing to Zulip
|
||||
|
||||
Welcome to the Zulip community!
|
||||
|
||||
## Community
|
||||
|
||||
The
|
||||
[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
|
||||
is the primary communication forum for the Zulip community. It is a good
|
||||
place to start whether you have a question, are a new contributor, are a new
|
||||
user, or anything else. Make sure to read the
|
||||
[community norms](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html#community-norms)
|
||||
before posting. The Zulip community is also governed by a
|
||||
[code of conduct](https://zulip.readthedocs.io/en/latest/code-of-conduct.html).
|
||||
|
||||
You can subscribe to zulip-devel-announce@googlegroups.com or our
|
||||
[Twitter](https://twitter.com/zulip) account for a lower traffic (~1
|
||||
email/month) way to hear about things like mentorship opportunities with Google
|
||||
Code-in, in-person sprints at conferences, and other opportunities to
|
||||
contribute.
|
||||
|
||||
## Ways to contribute
|
||||
|
||||
To make a code or documentation contribution, read our
|
||||
[step-by-step guide](#your-first-codebase-contribution) to getting
|
||||
started with the Zulip codebase. A small sample of the type of work that
|
||||
needs doing:
|
||||
* Bug squashing and feature development on our Python/Django
|
||||
[backend](https://github.com/zulip/zulip), web
|
||||
[frontend](https://github.com/zulip/zulip), React Native
|
||||
[mobile app](https://github.com/zulip/zulip-mobile), or Electron
|
||||
[desktop app](https://github.com/zulip/zulip-desktop).
|
||||
* Building out our
|
||||
[Python API and bots](https://github.com/zulip/python-zulip-api) framework.
|
||||
* [Writing an integration](https://zulipchat.com/api/integrations-overview).
|
||||
* Improving our [user](https://zulipchat.com/help/) or
|
||||
[developer](https://zulip.readthedocs.io/en/latest/) documentation.
|
||||
* [Reviewing code](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html)
|
||||
and manually testing pull requests.
|
||||
|
||||
**Non-code contributions**: Some of the most valuable ways to contribute
|
||||
don't require touching the codebase at all. We list a few of them below:
|
||||
|
||||
* [Reporting issues](#reporting-issues), including both feature requests and
|
||||
bug reports.
|
||||
* [Giving feedback](#user-feedback) if you are evaluating or using Zulip.
|
||||
* [Translating](https://zulip.readthedocs.io/en/latest/translating/translating.html)
|
||||
Zulip.
|
||||
* [Outreach](#zulip-outreach): Star us on GitHub, upvote us
|
||||
on product comparison sites, or write for [the Zulip blog](http://blog.zulip.org/).
|
||||
|
||||
## Your first (codebase) contribution
|
||||
|
||||
This section has a step by step guide to starting as a Zulip codebase
|
||||
contributor. It's long, but don't worry about doing all the steps perfectly;
|
||||
no one gets it right the first time, and there are a lot of people available
|
||||
to help.
|
||||
* First, make an account on the
|
||||
[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html),
|
||||
paying special attention to the community norms. If you'd like, introduce
|
||||
yourself in
|
||||
[#new members](https://chat.zulip.org/#narrow/stream/95-new-members), using
|
||||
your name as the topic. Bonus: tell us about your first impressions of
|
||||
Zulip, and anything that felt confusing/broken as you started using the
|
||||
product.
|
||||
* Read [What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
|
||||
* [Install the development environment](https://zulip.readthedocs.io/en/latest/development/overview.html),
|
||||
getting help in
|
||||
[#development help](https://chat.zulip.org/#narrow/stream/49-development-help)
|
||||
if you run into any troubles.
|
||||
* Read the
|
||||
[Zulip guide to Git](https://zulip.readthedocs.io/en/latest/git/index.html)
|
||||
and do the Git tutorial (coming soon) if you are unfamiliar with
|
||||
Git, getting help in
|
||||
[#git help](https://chat.zulip.org/#narrow/stream/44-git-help) if
|
||||
you run into any troubles. Be sure to check out the
|
||||
[extremely useful Zulip-specific tools page](https://zulip.readthedocs.io/en/latest/git/zulip-tools.html).
|
||||
* Sign the
|
||||
[Dropbox Contributor License Agreement](https://opensource.dropbox.com/cla/).
|
||||
|
||||
### Picking an issue
|
||||
|
||||
Now, you're ready to pick your first issue! There are hundreds of open issues
|
||||
in the main codebase alone. This section will help you find an issue to work
|
||||
on.
|
||||
|
||||
* If you're interested in
|
||||
[mobile](https://github.com/zulip/zulip-mobile/issues?q=is%3Aopen+is%3Aissue),
|
||||
[desktop](https://github.com/zulip/zulip-desktop/issues?q=is%3Aopen+is%3Aissue),
|
||||
or
|
||||
[bots](https://github.com/zulip/python-zulip-api/issues?q=is%3Aopen+is%3Aissue)
|
||||
development, check the respective links for open issues, or post in
|
||||
[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile),
|
||||
[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop), or
|
||||
[#integration](https://chat.zulip.org/#narrow/stream/127-integrations).
|
||||
* For the main server and web repository, we recommend browsing
|
||||
recently opened issues to look for issues you are confident you can
|
||||
fix correctly in a way that clearly communicates why your changes
|
||||
are the correct fix. Our GitHub workflow bot, zulipbot, limits
|
||||
users who have 0 commits merged to claiming a single issue labeled
|
||||
with "good first issue" or "help wanted".
|
||||
* We also partition all of our issues in the main repo into areas like
|
||||
admin, compose, emoji, hotkeys, i18n, onboarding, search, etc. Look
|
||||
through our [list of labels](https://github.com/zulip/zulip/labels), and
|
||||
click on some of the `area:` labels to see all the issues related to your
|
||||
areas of interest.
|
||||
* If the lists of issues are overwhelming, post in
|
||||
[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with a
|
||||
bit about your background and interests, and we'll help you out. The most
|
||||
important thing to say is whether you're looking for a backend (Python),
|
||||
frontend (JavaScript and TypeScript), mobile (React Native), desktop (Electron),
|
||||
documentation (English) or visual design (JavaScript/TypeScript + CSS) issue, and a
|
||||
bit about your programming experience and available time.
|
||||
|
||||
We also welcome suggestions of features that you feel would be valuable or
|
||||
changes that you feel would make Zulip a better open source project. If you
|
||||
have a new feature you'd like to add, we recommend you start by posting in
|
||||
[#new members](https://chat.zulip.org/#narrow/stream/95-new-members) with the
|
||||
feature idea and the problem that you're hoping to solve.
|
||||
|
||||
Other notes:
|
||||
* For a first pull request, it's better to aim for a smaller contribution
|
||||
than a bigger one. Many first contributions have fewer than 10 lines of
|
||||
changes (not counting changes to tests).
|
||||
* The full list of issues explicitly looking for a contributor can be
|
||||
found with the
|
||||
[good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
|
||||
and
|
||||
[help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
|
||||
labels. Avoid issues with the "difficult" label unless you
|
||||
understand why it is difficult and are confident you can resolve the
|
||||
issue correctly and completely. Issues without one of these labels
|
||||
are fair game if Tim has written a clear technical design proposal
|
||||
in the issue, or it is a bug that you can reproduce and you are
|
||||
confident you can fix the issue correctly.
|
||||
* For most new contributors, there's a lot to learn while making your first
|
||||
pull request. It's OK if it takes you a while; that's normal! You'll be
|
||||
able to work a lot faster as you build experience.
|
||||
|
||||
### Working on an issue
|
||||
|
||||
To work on an issue, claim it by adding a comment with `@zulipbot claim` to
|
||||
the issue thread. [Zulipbot](https://github.com/zulip/zulipbot) is a GitHub
|
||||
workflow bot; it will assign you to the issue and label the issue as "in
|
||||
progress". Some additional notes:
|
||||
|
||||
* You can only claim issues with the
|
||||
[good first issue](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
|
||||
or
|
||||
[help wanted](https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
|
||||
labels. Zulipbot will give you an error if you try to claim an issue
|
||||
without one of those labels.
|
||||
* You're encouraged to ask questions on how to best implement or debug your
|
||||
changes -- the Zulip maintainers are excited to answer questions to help
|
||||
you stay unblocked and working efficiently. You can ask questions on
|
||||
chat.zulip.org, or on the GitHub issue or pull request.
|
||||
* We encourage early pull requests for work in progress. Prefix the title of
|
||||
work in progress pull requests with `[WIP]`, and remove the prefix when
|
||||
you think it might be mergeable and want it to be reviewed.
|
||||
* After updating a PR, add a comment to the GitHub thread mentioning that it
|
||||
is ready for another review. GitHub only notifies maintainers of the
|
||||
changes when you post a comment, so if you don't, your PR will likely be
|
||||
neglected by accident!
|
||||
|
||||
### And beyond
|
||||
|
||||
A great place to look for a second issue is to look for issues with the same
|
||||
`area:` label as the last issue you resolved. You'll be able to reuse the
|
||||
work you did learning how that part of the codebase works. Also, the path to
|
||||
becoming a core developer often involves taking ownership of one of these area
|
||||
labels.
|
||||
|
||||
## What makes a great Zulip contributor?
|
||||
|
||||
Zulip runs a lot of [internship programs](#internship-programs), so we have
|
||||
a lot of experience with new contributors. In our experience, these are the
|
||||
best predictors of success:
|
||||
|
||||
* Posting good questions. This generally means explaining your current
|
||||
understanding, saying what you've done or tried so far, and including
|
||||
tracebacks or other error messages if appropriate.
|
||||
* Learning and practicing
|
||||
[Git commit discipline](https://zulip.readthedocs.io/en/latest/contributing/version-control.html#commit-discipline).
|
||||
* Submitting carefully tested code. This generally means checking your work
|
||||
through a combination of automated tests and manually clicking around the
|
||||
UI trying to find bugs in your work. See
|
||||
[things to look for](https://zulip.readthedocs.io/en/latest/contributing/code-reviewing.html#things-to-look-for)
|
||||
for additional ideas.
|
||||
* Posting
|
||||
[screenshots or GIFs](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
|
||||
for frontend changes.
|
||||
* Being responsive to feedback on pull requests. This means incorporating or
|
||||
responding to all suggested changes, and leaving a note if you won't be
|
||||
able to address things within a few days.
|
||||
* Being helpful and friendly on chat.zulip.org.
|
||||
|
||||
These are also the main criteria we use to select interns for all of our
|
||||
internship programs.
|
||||
|
||||
## Reporting issues
|
||||
|
||||
If you find an easily reproducible bug and/or are experienced in reporting
|
||||
bugs, feel free to just open an issue on the relevant project on GitHub.
|
||||
|
||||
If you have a feature request or are not yet sure what the underlying bug
|
||||
is, the best place to post issues is
|
||||
[#issues](https://chat.zulip.org/#narrow/stream/9-issues) (or
|
||||
[#mobile](https://chat.zulip.org/#narrow/stream/48-mobile) or
|
||||
[#desktop](https://chat.zulip.org/#narrow/stream/16-desktop)) on the
|
||||
[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html).
|
||||
This allows us to interactively figure out what is going on, let you know if
|
||||
a similar issue has already been opened, and collect any other information
|
||||
we need. Choose a 2-4 word topic that describes the issue, explain the issue
|
||||
and how to reproduce it if known, your browser/OS if relevant, and a
|
||||
[screenshot or screenGIF](https://zulip.readthedocs.io/en/latest/tutorials/screenshot-and-gif-software.html)
|
||||
if appropriate.
|
||||
|
||||
**Reporting security issues**. Please do not report security issues
|
||||
publicly, including on public streams on chat.zulip.org. You can email
|
||||
zulip-security@googlegroups.com. We create a CVE for every security issue.
|
||||
|
||||
## User feedback
|
||||
|
||||
Nearly every feature we develop starts with a user request. If you are part
|
||||
of a group that is either using or considering using Zulip, we would love to
|
||||
hear about your experience with the product. If you're not sure what to
|
||||
write, here are some questions we're always very curious to know the answer
|
||||
to:
|
||||
|
||||
* Evaluation: What is the process by which your organization chose or will
|
||||
choose a group chat product?
|
||||
* Pros and cons: What are the pros and cons of Zulip for your organization,
|
||||
and the pros and cons of other products you are evaluating?
|
||||
* Features: What are the features that are most important for your
|
||||
organization? In the best case scenario, what would your chat solution do
|
||||
for you?
|
||||
* Onboarding: If you remember it, what was your impression during your first
|
||||
few minutes of using Zulip? What did you notice, and how did you feel? Was
|
||||
there anything that stood out to you as confusing, or broken, or great?
|
||||
* Organization: What does your organization do? How big is the organization?
|
||||
A link to your organization's website?
|
||||
|
||||
## Internship programs
|
||||
|
||||
Zulip runs internship programs with
|
||||
[Outreachy](https://www.outreachy.org/),
|
||||
[Google Summer of Code (GSoC)](https://developers.google.com/open-source/gsoc/)
|
||||
[1], and the
|
||||
[MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram),
|
||||
and has in the past taken summer interns from Harvard, MIT, and
|
||||
Stanford.
|
||||
|
||||
While each third-party program has its own rules and requirements, the
|
||||
Zulip community's approaches all of these programs with these ideas in
|
||||
mind:
|
||||
* We try to make the application process as valuable for the applicant as
|
||||
possible. Expect high quality code reviews, a supportive community, and
|
||||
publicly viewable patches you can link to from your resume, regardless of
|
||||
whether you are selected.
|
||||
* To apply, you'll have to submit at least one pull request to a Zulip
|
||||
repository. Most students accepted to one of our programs have
|
||||
several merged pull requests (including at least one larger PR) by
|
||||
the time of the application deadline.
|
||||
* The main criteria we use is quality of your best contributions, and
|
||||
the bullets listed at
|
||||
[What makes a great Zulip contributor](#what-makes-a-great-zulip-contributor).
|
||||
Because we focus on evaluating your best work, it doesn't hurt your
|
||||
application to makes mistakes in your first few PRs as long as your
|
||||
work improves.
|
||||
|
||||
Zulip also participates in
|
||||
[Google Code-In](https://developers.google.com/open-source/gci/). Our
|
||||
selection criteria for Finalists and Grand Prize Winners is the same as our
|
||||
selection criteria for interns above.
|
||||
|
||||
Most of our interns end up sticking around the project long-term, and many
|
||||
quickly become core team members. We hope you apply!
|
||||
|
||||
### Google Summer of Code
|
||||
|
||||
GSoC is by far the largest of our internship programs (14 students in
|
||||
2017; 11 in 2018; 17 in 2019). While we don't control how many slots
|
||||
Google allocates to Zulip, we hope to mentor a similar number of
|
||||
students in future summers.
|
||||
|
||||
If you're reading this well before the application deadline and want
|
||||
to make your application strong, we recommend getting involved in the
|
||||
community and fixing issues in Zulip now. Having good contributions
|
||||
and building a reputation for doing good work is best way to have a
|
||||
strong application. About half of Zulip's GSoC students for Summer
|
||||
2017 had made significant contributions to the project by February
|
||||
2017, and about half had not. Our
|
||||
[GSoC project ideas page][gsoc-guide] has lots more details on how
|
||||
Zulip does GSoC, as well as project ideas (though the project idea
|
||||
list is maintained only during the GSoC application period, so if
|
||||
you're looking at some other time of year, the project list is likely
|
||||
out-of-date).
|
||||
|
||||
We also have in some past years run a Zulip Summer of Code (ZSoC)
|
||||
program for students who we didn't have enough slots to accept for
|
||||
GSoC but were able to find funding for. Student expectations are the
|
||||
same as with GSoC, and it has no separate application process; your
|
||||
GSoC application is your ZSoC application. If we'd like to select you
|
||||
for ZSoC, we'll contact you when the GSoC results are announced.
|
||||
|
||||
[gsoc-guide]: https://zulip.readthedocs.io/en/latest/overview/gsoc-ideas.html
|
||||
[gsoc-faq]: https://developers.google.com/open-source/gsoc/faq
|
||||
|
||||
[1] Formally, [GSoC isn't an internship][gsoc-faq], but it is similar
|
||||
enough that we're treating it as such for the purposes of this
|
||||
documentation.
|
||||
|
||||
## Zulip Outreach
|
||||
|
||||
**Upvoting Zulip**. Upvotes and reviews make a big difference in the public
|
||||
perception of projects like Zulip. We've collected a few sites below
|
||||
where we know Zulip has been discussed. Doing everything in the following
|
||||
list typically takes about 15 minutes.
|
||||
* Star us on GitHub. There are four main repositories:
|
||||
[server/web](https://github.com/zulip/zulip),
|
||||
[mobile](https://github.com/zulip/zulip-mobile),
|
||||
[desktop](https://github.com/zulip/zulip-desktop), and
|
||||
[Python API](https://github.com/zulip/python-zulip-api).
|
||||
* [Follow us](https://twitter.com/zulip) on Twitter.
|
||||
|
||||
For both of the following, you'll need to make an account on the site if you
|
||||
don't already have one.
|
||||
|
||||
* [Like Zulip](https://alternativeto.net/software/zulip-chat-server/) on
|
||||
AlternativeTo. We recommend upvoting a couple of other products you like
|
||||
as well, both to give back to their community, and since single-upvote
|
||||
accounts are generally given less weight. You can also
|
||||
[upvote Zulip](https://alternativeto.net/software/slack/) on their page
|
||||
for Slack.
|
||||
* [Add Zulip to your stack](https://stackshare.io/zulip) on StackShare, star
|
||||
it, and upvote the reasons why people like Zulip that you find most
|
||||
compelling. Again, we recommend adding a few other products that you like
|
||||
as well.
|
||||
|
||||
We have a doc with more detailed instructions and a few other sites, if you
|
||||
have been using Zulip for a while and want to contribute more.
|
||||
|
||||
**Blog posts**. Writing a blog post about your experiences with Zulip, or
|
||||
about a technical aspect of Zulip can be a great way to spread the word
|
||||
about Zulip.
|
||||
|
||||
We also occasionally [publish](http://blog.zulip.org/) longer form
|
||||
articles related to Zulip. Our posts typically get tens of thousands
|
||||
of views, and we always have good ideas for blog posts that we can
|
||||
outline but don't have time to write. If you are an experienced writer
|
||||
or copyeditor, send us a portfolio; we'd love to talk!
|
||||
@@ -1,23 +0,0 @@
|
||||
# To build run `docker build -f Dockerfile-postgresql .` from the root of the
|
||||
# zulip repo.
|
||||
|
||||
# Currently the postgres images do not support automatic upgrading of
|
||||
# the on-disk data in volumes. So the base image can not currently be upgraded
|
||||
# without users needing a manual pgdump and restore.
|
||||
|
||||
# Install hunspell, zulip stop words, and run zulip database
|
||||
# init.
|
||||
FROM postgres:10
|
||||
COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop
|
||||
COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
|
||||
COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql
|
||||
COPY scripts/setup/pgroonga-debian.asc /tmp
|
||||
RUN apt-key add /tmp/pgroonga-debian.asc \
|
||||
&& echo "deb http://packages.groonga.org/debian/ stretch main" > /etc/apt/sources.list.d/zulip.list \
|
||||
&& apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
|
||||
hunspell-en-us \
|
||||
postgresql-${PG_MAJOR}-pgroonga \
|
||||
&& ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \
|
||||
&& ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
204
LICENSE
204
LICENSE
@@ -1,203 +1,5 @@
|
||||
Copyright 2011-2018 Dropbox, Inc., Kandra Labs, Inc., and contributors
|
||||
Copyright <20> 2012-2013 Zulip, Inc.
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
This software is licensed under the Zulip Enterprise License Agreement.
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Zulip can be reached at support@zulip.com.
|
||||
|
||||
16
NOTICE
16
NOTICE
@@ -1,16 +0,0 @@
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this project except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
The software includes some works released by third parties under other
|
||||
free and open source licenses. Those works are redistributed under the
|
||||
license terms under which the works were received. For more details,
|
||||
see the ``docs/THIRDPARTY`` file included with this distribution.
|
||||
82
README.md
82
README.md
@@ -1,82 +0,0 @@
|
||||
# Zulip overview
|
||||
|
||||
Zulip is a powerful, open source group chat application that combines the
|
||||
immediacy of real-time chat with the productivity benefits of threaded
|
||||
conversations. Zulip is used by open source projects, Fortune 500 companies,
|
||||
large standards bodies, and others who need a real-time chat system that
|
||||
allows users to easily process hundreds or thousands of messages a day. With
|
||||
over 500 contributors merging over 500 commits a month, Zulip is also the
|
||||
largest and fastest growing open source group chat project.
|
||||
|
||||
[](https://circleci.com/gh/zulip/zulip/tree/master)
|
||||
[](https://codecov.io/gh/zulip/zulip/branch/master)
|
||||
[][mypy-coverage]
|
||||
[](https://github.com/zulip/zulip/releases/latest)
|
||||
[](https://zulip.readthedocs.io/en/latest/)
|
||||
[](https://chat.zulip.org)
|
||||
[](https://twitter.com/zulip)
|
||||
|
||||
[mypy-coverage]: https://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/
|
||||
|
||||
## Getting started
|
||||
|
||||
Click on the appropriate link below. If nothing seems to apply,
|
||||
join us on the
|
||||
[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html)
|
||||
and tell us what's up!
|
||||
|
||||
You might be interested in:
|
||||
|
||||
* **Contributing code**. Check out our
|
||||
[guide for new contributors](https://zulip.readthedocs.io/en/latest/overview/contributing.html)
|
||||
to get started. Zulip prides itself on maintaining a clean and
|
||||
well-tested codebase, and a stock of hundreds of
|
||||
[beginner-friendly issues][beginner-friendly].
|
||||
|
||||
* **Contributing non-code**.
|
||||
[Report an issue](https://zulip.readthedocs.io/en/latest/overview/contributing.html#reporting-issue),
|
||||
[translate](https://zulip.readthedocs.io/en/latest/translating/translating.html) Zulip
|
||||
into your language,
|
||||
[write](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach)
|
||||
for the Zulip blog, or
|
||||
[give us feedback](https://zulip.readthedocs.io/en/latest/overview/contributing.html#user-feedback). We
|
||||
would love to hear from you, even if you're just trying the product out.
|
||||
|
||||
* **Supporting Zulip**. Advocate for your organization to use Zulip, write a
|
||||
review in the mobile app stores, or
|
||||
[upvote Zulip](https://zulip.readthedocs.io/en/latest/overview/contributing.html#zulip-outreach) on
|
||||
product comparison sites.
|
||||
|
||||
* **Checking Zulip out**. The best way to see Zulip in action is to drop by
|
||||
the
|
||||
[Zulip community server](https://zulip.readthedocs.io/en/latest/contributing/chat-zulip-org.html). We
|
||||
also recommend reading Zulip for
|
||||
[open source](https://zulipchat.com/for/open-source/), Zulip for
|
||||
[companies](https://zulipchat.com/for/companies/), or Zulip for
|
||||
[working groups and part time communities](https://zulipchat.com/for/working-groups-and-communities/).
|
||||
|
||||
* **Running a Zulip server**. Use a preconfigured [Digital Ocean droplet](https://marketplace.digitalocean.com/apps/zulip),
|
||||
[install Zulip](https://zulip.readthedocs.io/en/stable/production/install.html)
|
||||
directly, or use Zulip's
|
||||
experimental [Docker image](https://zulip.readthedocs.io/en/latest/production/deployment.html#zulip-in-docker).
|
||||
Commercial support is available; see <https://zulipchat.com/plans> for details.
|
||||
|
||||
* **Using Zulip without setting up a server**. <https://zulipchat.com> offers
|
||||
free and commercial hosting.
|
||||
|
||||
* **Applying for a Zulip internship**. Zulip runs internship programs with
|
||||
[Outreachy](https://www.outreachy.org/),
|
||||
[Google Summer of Code](https://developers.google.com/open-source/gsoc/),
|
||||
and the
|
||||
[MIT Externship program](https://alum.mit.edu/students/NetworkwithAlumni/ExternshipProgram). Zulip
|
||||
also participates in
|
||||
[Google Code-In](https://developers.google.com/open-source/gci/). More
|
||||
information is available
|
||||
[here](https://zulip.readthedocs.io/en/latest/overview/contributing.html#internship-programs).
|
||||
|
||||
You may also be interested in reading our [blog](http://blog.zulip.org/) or
|
||||
following us on [twitter](https://twitter.com/zulip).
|
||||
Zulip is distributed under the
|
||||
[Apache 2.0](https://github.com/zulip/zulip/blob/master/LICENSE) license.
|
||||
|
||||
[beginner-friendly]: https://github.com/zulip/zulip/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22
|
||||
195
Vagrantfile
vendored
195
Vagrantfile
vendored
@@ -1,195 +0,0 @@
|
||||
# -*- mode: ruby -*-
|
||||
|
||||
VAGRANTFILE_API_VERSION = "2"
|
||||
|
||||
def command?(name)
|
||||
`which #{name} > /dev/null 2>&1`
|
||||
$?.success?
|
||||
end
|
||||
|
||||
if Vagrant::VERSION == "1.8.7" then
|
||||
path = `which curl`
|
||||
if path.include?('/opt/vagrant/embedded/bin/curl') then
|
||||
puts "In Vagrant 1.8.7, curl is broken. Please use Vagrant 2.0.2 "\
|
||||
"or run 'sudo rm -f /opt/vagrant/embedded/bin/curl' to fix the "\
|
||||
"issue before provisioning. See "\
|
||||
"https://github.com/mitchellh/vagrant/issues/7997 "\
|
||||
"for reference."
|
||||
exit
|
||||
end
|
||||
end
|
||||
|
||||
# Workaround: Vagrant removed the atlas.hashicorp.com to
|
||||
# vagrantcloud.com redirect in February 2018. The value of
|
||||
# DEFAULT_SERVER_URL in Vagrant versions less than 1.9.3 is
|
||||
# atlas.hashicorp.com, which means that removal broke the fetching and
|
||||
# updating of boxes (since the old URL doesn't work). See
|
||||
# https://github.com/hashicorp/vagrant/issues/9442
|
||||
if Vagrant::DEFAULT_SERVER_URL == "atlas.hashicorp.com"
|
||||
Vagrant::DEFAULT_SERVER_URL.replace('https://vagrantcloud.com')
|
||||
end
|
||||
|
||||
# Monkey patch https://github.com/hashicorp/vagrant/pull/10879 so we
|
||||
# can fall back to another provider if docker is not installed.
|
||||
begin
|
||||
require Vagrant.source_root.join("plugins", "providers", "docker", "provider")
|
||||
rescue LoadError
|
||||
else
|
||||
VagrantPlugins::DockerProvider::Provider.class_eval do
|
||||
method(:usable?).owner == singleton_class or def self.usable?(raise_error=false)
|
||||
VagrantPlugins::DockerProvider::Driver.new.execute("docker", "version")
|
||||
true
|
||||
rescue Vagrant::Errors::CommandUnavailable, VagrantPlugins::DockerProvider::Errors::ExecuteError
|
||||
raise if raise_error
|
||||
return false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
||||
|
||||
# The Zulip development environment runs on 9991 on the guest.
|
||||
host_port = 9991
|
||||
http_proxy = https_proxy = no_proxy = nil
|
||||
host_ip_addr = "127.0.0.1"
|
||||
|
||||
# System settings for the virtual machine.
|
||||
vm_num_cpus = "2"
|
||||
vm_memory = "2048"
|
||||
|
||||
ubuntu_mirror = ""
|
||||
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
config.vm.synced_folder ".", "/srv/zulip"
|
||||
|
||||
vagrant_config_file = ENV['HOME'] + "/.zulip-vagrant-config"
|
||||
if File.file?(vagrant_config_file)
|
||||
IO.foreach(vagrant_config_file) do |line|
|
||||
line.chomp!
|
||||
key, value = line.split(nil, 2)
|
||||
case key
|
||||
when /^([#;]|$)/; # ignore comments
|
||||
when "HTTP_PROXY"; http_proxy = value
|
||||
when "HTTPS_PROXY"; https_proxy = value
|
||||
when "NO_PROXY"; no_proxy = value
|
||||
when "HOST_PORT"; host_port = value.to_i
|
||||
when "HOST_IP_ADDR"; host_ip_addr = value
|
||||
when "GUEST_CPUS"; vm_num_cpus = value
|
||||
when "GUEST_MEMORY_MB"; vm_memory = value
|
||||
when "UBUNTU_MIRROR"; ubuntu_mirror = value
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if Vagrant.has_plugin?("vagrant-proxyconf")
|
||||
if !http_proxy.nil?
|
||||
config.proxy.http = http_proxy
|
||||
end
|
||||
if !https_proxy.nil?
|
||||
config.proxy.https = https_proxy
|
||||
end
|
||||
if !no_proxy.nil?
|
||||
config.proxy.no_proxy = no_proxy
|
||||
end
|
||||
elsif !http_proxy.nil? or !https_proxy.nil?
|
||||
# This prints twice due to https://github.com/hashicorp/vagrant/issues/7504
|
||||
# We haven't figured out a workaround.
|
||||
puts 'You have specified value for proxy in ~/.zulip-vagrant-config file but did not ' \
|
||||
'install the vagrant-proxyconf plugin. To install it, run `vagrant plugin install ' \
|
||||
'vagrant-proxyconf` in a terminal. This error will appear twice.'
|
||||
exit
|
||||
end
|
||||
|
||||
config.vm.network "forwarded_port", guest: 9991, host: host_port, host_ip: host_ip_addr
|
||||
config.vm.network "forwarded_port", guest: 9994, host: host_port + 3, host_ip: host_ip_addr
|
||||
# Specify Docker provider before VirtualBox provider so it's preferred.
|
||||
config.vm.provider "docker" do |d, override|
|
||||
d.build_dir = File.join(__dir__, "tools", "setup", "dev-vagrant-docker")
|
||||
d.build_args = ["--build-arg", "VAGRANT_UID=#{Process.uid}"]
|
||||
if !ubuntu_mirror.empty?
|
||||
d.build_args += ["--build-arg", "UBUNTU_MIRROR=#{ubuntu_mirror}"]
|
||||
end
|
||||
d.has_ssh = true
|
||||
d.create_args = ["--ulimit", "nofile=1024:65536"]
|
||||
end
|
||||
|
||||
config.vm.provider "virtualbox" do |vb, override|
|
||||
override.vm.box = "ubuntu/bionic64"
|
||||
# An unnecessary log file gets generated when running vagrant up for the
|
||||
# first time with the Ubuntu Bionic box. This looks like it is being
|
||||
# caused upstream by the base box containing a Vagrantfile with a similar
|
||||
# line to the one below.
|
||||
# see https://github.com/hashicorp/vagrant/issues/9425
|
||||
vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ]
|
||||
# It's possible we can get away with just 1.5GB; more testing needed
|
||||
vb.memory = vm_memory
|
||||
vb.cpus = vm_num_cpus
|
||||
end
|
||||
|
||||
$provision_script = <<SCRIPT
|
||||
set -x
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
# Code should go here, rather than tools/provision, only if it is
|
||||
# something that we don't want to happen when running provision in a
|
||||
# development environment not using Vagrant.
|
||||
|
||||
# Set the Ubuntu mirror
|
||||
[ ! '#{ubuntu_mirror}' ] || sudo sed -i 's|http://\\(\\w*\\.\\)*archive\\.ubuntu\\.com/ubuntu/\\? |#{ubuntu_mirror} |' /etc/apt/sources.list
|
||||
|
||||
# Set the MOTD on the system to have Zulip instructions
|
||||
sudo ln -nsf /srv/zulip/tools/setup/dev-motd /etc/update-motd.d/99-zulip-dev
|
||||
sudo rm -f /etc/update-motd.d/10-help-text
|
||||
sudo dpkg --purge landscape-client landscape-common ubuntu-release-upgrader-core update-manager-core update-notifier-common ubuntu-server
|
||||
sudo dpkg-divert --add --rename /etc/default/motd-news
|
||||
sudo sh -c 'echo ENABLED=0 > /etc/default/motd-news'
|
||||
|
||||
# If the host is running SELinux remount the /sys/fs/selinux directory as read only,
|
||||
# needed for apt-get to work.
|
||||
if [ -d "/sys/fs/selinux" ]; then
|
||||
sudo mount -o remount,ro /sys/fs/selinux
|
||||
fi
|
||||
|
||||
# Set default locale, this prevents errors if the user has another locale set.
|
||||
if ! grep -q 'LC_ALL=en_US.UTF-8' /etc/default/locale; then
|
||||
echo "LC_ALL=en_US.UTF-8" | sudo tee -a /etc/default/locale
|
||||
fi
|
||||
|
||||
# Set an environment variable, so that we won't print the virtualenv
|
||||
# shell warning (it'll be wrong, since the shell is dying anyway)
|
||||
export SKIP_VENV_SHELL_WARNING=1
|
||||
|
||||
# End `set -x`, so that the end of provision doesn't look like an error
|
||||
# message after a successful run.
|
||||
set +x
|
||||
|
||||
# Check if the zulip directory is writable
|
||||
if [ ! -w /srv/zulip ]; then
|
||||
echo "The vagrant user is unable to write to the zulip directory."
|
||||
echo "To fix this, run the following commands on the host machine:"
|
||||
# sudo is required since our uid is not 1000
|
||||
echo ' vagrant halt -f'
|
||||
echo ' rm -rf /PATH/TO/ZULIP/CLONE/.vagrant'
|
||||
echo ' sudo chown -R 1000:$(id -g) /PATH/TO/ZULIP/CLONE'
|
||||
echo "Replace /PATH/TO/ZULIP/CLONE with the path to where zulip code is cloned."
|
||||
echo "You can resume setting up your vagrant environment by running:"
|
||||
echo " vagrant up"
|
||||
exit 1
|
||||
fi
|
||||
# Provision the development environment
|
||||
ln -nsf /srv/zulip ~/zulip
|
||||
/srv/zulip/tools/provision
|
||||
|
||||
# Run any custom provision hooks the user has configured
|
||||
if [ -f /srv/zulip/tools/custom_provision ]; then
|
||||
chmod +x /srv/zulip/tools/custom_provision
|
||||
/srv/zulip/tools/custom_provision
|
||||
fi
|
||||
SCRIPT
|
||||
|
||||
config.vm.provision "shell",
|
||||
# We want provision to be run with the permissions of the vagrant user.
|
||||
privileged: false,
|
||||
inline: $provision_script
|
||||
end
|
||||
@@ -1,541 +0,0 @@
|
||||
import time
|
||||
from collections import OrderedDict, defaultdict
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
from typing import Callable, Dict, List, \
|
||||
Optional, Tuple, Type, Union
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from django.db.models import F
|
||||
|
||||
from analytics.models import BaseCount, \
|
||||
FillState, InstallationCount, RealmCount, StreamCount, \
|
||||
UserCount, installation_epoch, last_successful_fill
|
||||
from zerver.lib.logging_util import log_to_file
|
||||
from zerver.lib.timestamp import ceiling_to_day, \
|
||||
ceiling_to_hour, floor_to_hour, verify_UTC
|
||||
from zerver.models import Message, Realm, RealmAuditLog, \
|
||||
Stream, UserActivityInterval, UserProfile, models
|
||||
|
||||
## Logging setup ##
|
||||
|
||||
logger = logging.getLogger('zulip.management')
|
||||
log_to_file(logger, settings.ANALYTICS_LOG_PATH)
|
||||
|
||||
# You can't subtract timedelta.max from a datetime, so use this instead
|
||||
TIMEDELTA_MAX = timedelta(days=365*1000)
|
||||
|
||||
## Class definitions ##
|
||||
|
||||
class CountStat:
|
||||
HOUR = 'hour'
|
||||
DAY = 'day'
|
||||
FREQUENCIES = frozenset([HOUR, DAY])
|
||||
|
||||
def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
|
||||
interval: Optional[timedelta]=None) -> None:
|
||||
self.property = property
|
||||
self.data_collector = data_collector
|
||||
# might have to do something different for bitfields
|
||||
if frequency not in self.FREQUENCIES:
|
||||
raise AssertionError("Unknown frequency: %s" % (frequency,))
|
||||
self.frequency = frequency
|
||||
if interval is not None:
|
||||
self.interval = interval
|
||||
elif frequency == CountStat.HOUR:
|
||||
self.interval = timedelta(hours=1)
|
||||
else: # frequency == CountStat.DAY
|
||||
self.interval = timedelta(days=1)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<CountStat: %s>" % (self.property,)
|
||||
|
||||
class LoggingCountStat(CountStat):
|
||||
def __init__(self, property: str, output_table: Type[BaseCount], frequency: str) -> None:
|
||||
CountStat.__init__(self, property, DataCollector(output_table, None), frequency)
|
||||
|
||||
class DependentCountStat(CountStat):
|
||||
def __init__(self, property: str, data_collector: 'DataCollector', frequency: str,
|
||||
interval: Optional[timedelta]=None, dependencies: List[str]=[]) -> None:
|
||||
CountStat.__init__(self, property, data_collector, frequency, interval=interval)
|
||||
self.dependencies = dependencies
|
||||
|
||||
class DataCollector:
|
||||
def __init__(self, output_table: Type[BaseCount],
|
||||
pull_function: Optional[Callable[[str, datetime, datetime], int]]) -> None:
|
||||
self.output_table = output_table
|
||||
self.pull_function = pull_function
|
||||
|
||||
## CountStat-level operations ##
|
||||
|
||||
def process_count_stat(stat: CountStat, fill_to_time: datetime) -> None:
|
||||
if stat.frequency == CountStat.HOUR:
|
||||
time_increment = timedelta(hours=1)
|
||||
elif stat.frequency == CountStat.DAY:
|
||||
time_increment = timedelta(days=1)
|
||||
else:
|
||||
raise AssertionError("Unknown frequency: %s" % (stat.frequency,))
|
||||
|
||||
verify_UTC(fill_to_time)
|
||||
if floor_to_hour(fill_to_time) != fill_to_time:
|
||||
raise ValueError("fill_to_time must be on an hour boundary: %s" % (fill_to_time,))
|
||||
|
||||
fill_state = FillState.objects.filter(property=stat.property).first()
|
||||
if fill_state is None:
|
||||
currently_filled = installation_epoch()
|
||||
fill_state = FillState.objects.create(property=stat.property,
|
||||
end_time=currently_filled,
|
||||
state=FillState.DONE)
|
||||
logger.info("INITIALIZED %s %s" % (stat.property, currently_filled))
|
||||
elif fill_state.state == FillState.STARTED:
|
||||
logger.info("UNDO START %s %s" % (stat.property, fill_state.end_time))
|
||||
do_delete_counts_at_hour(stat, fill_state.end_time)
|
||||
currently_filled = fill_state.end_time - time_increment
|
||||
do_update_fill_state(fill_state, currently_filled, FillState.DONE)
|
||||
logger.info("UNDO DONE %s" % (stat.property,))
|
||||
elif fill_state.state == FillState.DONE:
|
||||
currently_filled = fill_state.end_time
|
||||
else:
|
||||
raise AssertionError("Unknown value for FillState.state: %s." % (fill_state.state,))
|
||||
|
||||
if isinstance(stat, DependentCountStat):
|
||||
for dependency in stat.dependencies:
|
||||
dependency_fill_time = last_successful_fill(dependency)
|
||||
if dependency_fill_time is None:
|
||||
logger.warning("DependentCountStat %s run before dependency %s." %
|
||||
(stat.property, dependency))
|
||||
return
|
||||
fill_to_time = min(fill_to_time, dependency_fill_time)
|
||||
|
||||
currently_filled = currently_filled + time_increment
|
||||
while currently_filled <= fill_to_time:
|
||||
logger.info("START %s %s" % (stat.property, currently_filled))
|
||||
start = time.time()
|
||||
do_update_fill_state(fill_state, currently_filled, FillState.STARTED)
|
||||
do_fill_count_stat_at_hour(stat, currently_filled)
|
||||
do_update_fill_state(fill_state, currently_filled, FillState.DONE)
|
||||
end = time.time()
|
||||
currently_filled = currently_filled + time_increment
|
||||
logger.info("DONE %s (%dms)" % (stat.property, (end-start)*1000))
|
||||
|
||||
def do_update_fill_state(fill_state: FillState, end_time: datetime, state: int) -> None:
|
||||
fill_state.end_time = end_time
|
||||
fill_state.state = state
|
||||
fill_state.save()
|
||||
|
||||
# We assume end_time is valid (e.g. is on a day or hour boundary as appropriate)
|
||||
# and is timezone aware. It is the caller's responsibility to enforce this!
|
||||
def do_fill_count_stat_at_hour(stat: CountStat, end_time: datetime) -> None:
|
||||
start_time = end_time - stat.interval
|
||||
if not isinstance(stat, LoggingCountStat):
|
||||
timer = time.time()
|
||||
assert(stat.data_collector.pull_function is not None)
|
||||
rows_added = stat.data_collector.pull_function(stat.property, start_time, end_time)
|
||||
logger.info("%s run pull_function (%dms/%sr)" %
|
||||
(stat.property, (time.time()-timer)*1000, rows_added))
|
||||
do_aggregate_to_summary_table(stat, end_time)
|
||||
|
||||
def do_delete_counts_at_hour(stat: CountStat, end_time: datetime) -> None:
|
||||
if isinstance(stat, LoggingCountStat):
|
||||
InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
if stat.data_collector.output_table in [UserCount, StreamCount]:
|
||||
RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
else:
|
||||
UserCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
StreamCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
RealmCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
InstallationCount.objects.filter(property=stat.property, end_time=end_time).delete()
|
||||
|
||||
def do_aggregate_to_summary_table(stat: CountStat, end_time: datetime) -> None:
|
||||
cursor = connection.cursor()
|
||||
|
||||
# Aggregate into RealmCount
|
||||
output_table = stat.data_collector.output_table
|
||||
if output_table in (UserCount, StreamCount):
|
||||
realmcount_query = """
|
||||
INSERT INTO analytics_realmcount
|
||||
(realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_realm.id, COALESCE(sum(%(output_table)s.value), 0), '%(property)s',
|
||||
%(output_table)s.subgroup, %%(end_time)s
|
||||
FROM zerver_realm
|
||||
JOIN %(output_table)s
|
||||
ON
|
||||
zerver_realm.id = %(output_table)s.realm_id
|
||||
WHERE
|
||||
%(output_table)s.property = '%(property)s' AND
|
||||
%(output_table)s.end_time = %%(end_time)s
|
||||
GROUP BY zerver_realm.id, %(output_table)s.subgroup
|
||||
""" % {'output_table': output_table._meta.db_table,
|
||||
'property': stat.property}
|
||||
start = time.time()
|
||||
cursor.execute(realmcount_query, {'end_time': end_time})
|
||||
end = time.time()
|
||||
logger.info("%s RealmCount aggregation (%dms/%sr)" % (
|
||||
stat.property, (end - start) * 1000, cursor.rowcount))
|
||||
|
||||
# Aggregate into InstallationCount
|
||||
installationcount_query = """
|
||||
INSERT INTO analytics_installationcount
|
||||
(value, property, subgroup, end_time)
|
||||
SELECT
|
||||
sum(value), '%(property)s', analytics_realmcount.subgroup, %%(end_time)s
|
||||
FROM analytics_realmcount
|
||||
WHERE
|
||||
property = '%(property)s' AND
|
||||
end_time = %%(end_time)s
|
||||
GROUP BY analytics_realmcount.subgroup
|
||||
""" % {'property': stat.property}
|
||||
start = time.time()
|
||||
cursor.execute(installationcount_query, {'end_time': end_time})
|
||||
end = time.time()
|
||||
logger.info("%s InstallationCount aggregation (%dms/%sr)" % (
|
||||
stat.property, (end - start) * 1000, cursor.rowcount))
|
||||
cursor.close()
|
||||
|
||||
## Utility functions called from outside counts.py ##
|
||||
|
||||
# called from zerver/lib/actions.py; should not throw any errors
|
||||
def do_increment_logging_stat(zerver_object: Union[Realm, UserProfile, Stream], stat: CountStat,
|
||||
subgroup: Optional[Union[str, int, bool]], event_time: datetime,
|
||||
increment: int=1) -> None:
|
||||
table = stat.data_collector.output_table
|
||||
if table == RealmCount:
|
||||
id_args = {'realm': zerver_object}
|
||||
elif table == UserCount:
|
||||
id_args = {'realm': zerver_object.realm, 'user': zerver_object}
|
||||
else: # StreamCount
|
||||
id_args = {'realm': zerver_object.realm, 'stream': zerver_object}
|
||||
|
||||
if stat.frequency == CountStat.DAY:
|
||||
end_time = ceiling_to_day(event_time)
|
||||
else: # CountStat.HOUR:
|
||||
end_time = ceiling_to_hour(event_time)
|
||||
|
||||
row, created = table.objects.get_or_create(
|
||||
property=stat.property, subgroup=subgroup, end_time=end_time,
|
||||
defaults={'value': increment}, **id_args)
|
||||
if not created:
|
||||
row.value = F('value') + increment
|
||||
row.save(update_fields=['value'])
|
||||
|
||||
def do_drop_all_analytics_tables() -> None:
|
||||
UserCount.objects.all().delete()
|
||||
StreamCount.objects.all().delete()
|
||||
RealmCount.objects.all().delete()
|
||||
InstallationCount.objects.all().delete()
|
||||
FillState.objects.all().delete()
|
||||
|
||||
def do_drop_single_stat(property: str) -> None:
|
||||
UserCount.objects.filter(property=property).delete()
|
||||
StreamCount.objects.filter(property=property).delete()
|
||||
RealmCount.objects.filter(property=property).delete()
|
||||
InstallationCount.objects.filter(property=property).delete()
|
||||
FillState.objects.filter(property=property).delete()
|
||||
|
||||
## DataCollector-level operations ##
|
||||
|
||||
def do_pull_by_sql_query(property: str, start_time: datetime, end_time: datetime, query: str,
|
||||
group_by: Optional[Tuple[models.Model, str]]) -> int:
|
||||
if group_by is None:
|
||||
subgroup = 'NULL'
|
||||
group_by_clause = ''
|
||||
else:
|
||||
subgroup = '%s.%s' % (group_by[0]._meta.db_table, group_by[1])
|
||||
group_by_clause = ', ' + subgroup
|
||||
|
||||
# We do string replacement here because cursor.execute will reject a
|
||||
# group_by_clause given as a param.
|
||||
# We pass in the datetimes as params to cursor.execute so that we don't have to
|
||||
# think about how to convert python datetimes to SQL datetimes.
|
||||
query_ = query % {'property': property, 'subgroup': subgroup,
|
||||
'group_by_clause': group_by_clause}
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(query_, {'time_start': start_time, 'time_end': end_time})
|
||||
rowcount = cursor.rowcount
|
||||
cursor.close()
|
||||
return rowcount
|
||||
|
||||
def sql_data_collector(output_table: Type[BaseCount], query: str,
|
||||
group_by: Optional[Tuple[models.Model, str]]) -> DataCollector:
|
||||
def pull_function(property: str, start_time: datetime, end_time: datetime) -> int:
|
||||
return do_pull_by_sql_query(property, start_time, end_time, query, group_by)
|
||||
return DataCollector(output_table, pull_function)
|
||||
|
||||
def do_pull_minutes_active(property: str, start_time: datetime, end_time: datetime) -> int:
|
||||
user_activity_intervals = UserActivityInterval.objects.filter(
|
||||
end__gt=start_time, start__lt=end_time
|
||||
).select_related(
|
||||
'user_profile'
|
||||
).values_list(
|
||||
'user_profile_id', 'user_profile__realm_id', 'start', 'end')
|
||||
|
||||
seconds_active = defaultdict(float) # type: Dict[Tuple[int, int], float]
|
||||
for user_id, realm_id, interval_start, interval_end in user_activity_intervals:
|
||||
start = max(start_time, interval_start)
|
||||
end = min(end_time, interval_end)
|
||||
seconds_active[(user_id, realm_id)] += (end - start).total_seconds()
|
||||
|
||||
rows = [UserCount(user_id=ids[0], realm_id=ids[1], property=property,
|
||||
end_time=end_time, value=int(seconds // 60))
|
||||
for ids, seconds in seconds_active.items() if seconds >= 60]
|
||||
UserCount.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
count_message_by_user_query = """
|
||||
INSERT INTO analytics_usercount
|
||||
(user_id, realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_userprofile.id, zerver_userprofile.realm_id, count(*),
|
||||
'%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_userprofile
|
||||
JOIN zerver_message
|
||||
ON
|
||||
zerver_userprofile.id = zerver_message.sender_id
|
||||
WHERE
|
||||
zerver_userprofile.date_joined < %%(time_end)s AND
|
||||
zerver_message.date_sent >= %%(time_start)s AND
|
||||
zerver_message.date_sent < %%(time_end)s
|
||||
GROUP BY zerver_userprofile.id %(group_by_clause)s
|
||||
"""
|
||||
|
||||
# Note: ignores the group_by / group_by_clause.
|
||||
count_message_type_by_user_query = """
|
||||
INSERT INTO analytics_usercount
|
||||
(realm_id, user_id, value, property, subgroup, end_time)
|
||||
SELECT realm_id, id, SUM(count) AS value, '%(property)s', message_type, %%(time_end)s
|
||||
FROM
|
||||
(
|
||||
SELECT zerver_userprofile.realm_id, zerver_userprofile.id, count(*),
|
||||
CASE WHEN
|
||||
zerver_recipient.type = 1 THEN 'private_message'
|
||||
WHEN
|
||||
zerver_recipient.type = 3 THEN 'huddle_message'
|
||||
WHEN
|
||||
zerver_stream.invite_only = TRUE THEN 'private_stream'
|
||||
ELSE 'public_stream'
|
||||
END
|
||||
message_type
|
||||
|
||||
FROM zerver_userprofile
|
||||
JOIN zerver_message
|
||||
ON
|
||||
zerver_userprofile.id = zerver_message.sender_id AND
|
||||
zerver_message.date_sent >= %%(time_start)s AND
|
||||
zerver_message.date_sent < %%(time_end)s
|
||||
JOIN zerver_recipient
|
||||
ON
|
||||
zerver_message.recipient_id = zerver_recipient.id
|
||||
LEFT JOIN zerver_stream
|
||||
ON
|
||||
zerver_recipient.type_id = zerver_stream.id
|
||||
GROUP BY
|
||||
zerver_userprofile.realm_id, zerver_userprofile.id,
|
||||
zerver_recipient.type, zerver_stream.invite_only
|
||||
) AS subquery
|
||||
GROUP BY realm_id, id, message_type
|
||||
"""
|
||||
|
||||
# This query joins to the UserProfile table since all current queries that
|
||||
# use this also subgroup on UserProfile.is_bot. If in the future there is a
|
||||
# stat that counts messages by stream and doesn't need the UserProfile
|
||||
# table, consider writing a new query for efficiency.
|
||||
count_message_by_stream_query = """
|
||||
INSERT INTO analytics_streamcount
|
||||
(stream_id, realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_stream.id, zerver_stream.realm_id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_stream
|
||||
JOIN zerver_recipient
|
||||
ON
|
||||
zerver_stream.id = zerver_recipient.type_id
|
||||
JOIN zerver_message
|
||||
ON
|
||||
zerver_recipient.id = zerver_message.recipient_id
|
||||
JOIN zerver_userprofile
|
||||
ON
|
||||
zerver_message.sender_id = zerver_userprofile.id
|
||||
WHERE
|
||||
zerver_stream.date_created < %%(time_end)s AND
|
||||
zerver_recipient.type = 2 AND
|
||||
zerver_message.date_sent >= %%(time_start)s AND
|
||||
zerver_message.date_sent < %%(time_end)s
|
||||
GROUP BY zerver_stream.id %(group_by_clause)s
|
||||
"""
|
||||
|
||||
# Hardcodes the query needed by active_users:is_bot:day, since that is
|
||||
# currently the only stat that uses this.
|
||||
count_user_by_realm_query = """
|
||||
INSERT INTO analytics_realmcount
|
||||
(realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_realm.id, count(*),'%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_realm
|
||||
JOIN zerver_userprofile
|
||||
ON
|
||||
zerver_realm.id = zerver_userprofile.realm_id
|
||||
WHERE
|
||||
zerver_realm.date_created < %%(time_end)s AND
|
||||
zerver_userprofile.date_joined >= %%(time_start)s AND
|
||||
zerver_userprofile.date_joined < %%(time_end)s AND
|
||||
zerver_userprofile.is_active = TRUE
|
||||
GROUP BY zerver_realm.id %(group_by_clause)s
|
||||
"""
|
||||
|
||||
# Currently hardcodes the query needed for active_users_audit:is_bot:day.
|
||||
# Assumes that a user cannot have two RealmAuditLog entries with the same event_time and
|
||||
# event_type in [RealmAuditLog.USER_CREATED, USER_DEACTIVATED, etc].
|
||||
# In particular, it's important to ensure that migrations don't cause that to happen.
|
||||
check_realmauditlog_by_user_query = """
|
||||
INSERT INTO analytics_usercount
|
||||
(user_id, realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
ral1.modified_user_id, ral1.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_realmauditlog ral1
|
||||
JOIN (
|
||||
SELECT modified_user_id, max(event_time) AS max_event_time
|
||||
FROM zerver_realmauditlog
|
||||
WHERE
|
||||
event_type in ({user_created}, {user_activated}, {user_deactivated}, {user_reactivated}) AND
|
||||
event_time < %%(time_end)s
|
||||
GROUP BY modified_user_id
|
||||
) ral2
|
||||
ON
|
||||
ral1.event_time = max_event_time AND
|
||||
ral1.modified_user_id = ral2.modified_user_id
|
||||
JOIN zerver_userprofile
|
||||
ON
|
||||
ral1.modified_user_id = zerver_userprofile.id
|
||||
WHERE
|
||||
ral1.event_type in ({user_created}, {user_activated}, {user_reactivated})
|
||||
""".format(user_created=RealmAuditLog.USER_CREATED,
|
||||
user_activated=RealmAuditLog.USER_ACTIVATED,
|
||||
user_deactivated=RealmAuditLog.USER_DEACTIVATED,
|
||||
user_reactivated=RealmAuditLog.USER_REACTIVATED)
|
||||
|
||||
check_useractivityinterval_by_user_query = """
|
||||
INSERT INTO analytics_usercount
|
||||
(user_id, realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_userprofile.id, zerver_userprofile.realm_id, 1, '%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_userprofile
|
||||
JOIN zerver_useractivityinterval
|
||||
ON
|
||||
zerver_userprofile.id = zerver_useractivityinterval.user_profile_id
|
||||
WHERE
|
||||
zerver_useractivityinterval.end >= %%(time_start)s AND
|
||||
zerver_useractivityinterval.start < %%(time_end)s
|
||||
GROUP BY zerver_userprofile.id %(group_by_clause)s
|
||||
"""
|
||||
|
||||
count_realm_active_humans_query = """
|
||||
INSERT INTO analytics_realmcount
|
||||
(realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
usercount1.realm_id, count(*), '%(property)s', NULL, %%(time_end)s
|
||||
FROM (
|
||||
SELECT realm_id, user_id
|
||||
FROM analytics_usercount
|
||||
WHERE
|
||||
property = 'active_users_audit:is_bot:day' AND
|
||||
subgroup = 'false' AND
|
||||
end_time = %%(time_end)s
|
||||
) usercount1
|
||||
JOIN (
|
||||
SELECT realm_id, user_id
|
||||
FROM analytics_usercount
|
||||
WHERE
|
||||
property = '15day_actives::day' AND
|
||||
end_time = %%(time_end)s
|
||||
) usercount2
|
||||
ON
|
||||
usercount1.user_id = usercount2.user_id
|
||||
GROUP BY usercount1.realm_id
|
||||
"""
|
||||
|
||||
# Currently unused and untested
|
||||
count_stream_by_realm_query = """
|
||||
INSERT INTO analytics_realmcount
|
||||
(realm_id, value, property, subgroup, end_time)
|
||||
SELECT
|
||||
zerver_realm.id, count(*), '%(property)s', %(subgroup)s, %%(time_end)s
|
||||
FROM zerver_realm
|
||||
JOIN zerver_stream
|
||||
ON
|
||||
zerver_realm.id = zerver_stream.realm_id AND
|
||||
WHERE
|
||||
zerver_realm.date_created < %%(time_end)s AND
|
||||
zerver_stream.date_created >= %%(time_start)s AND
|
||||
zerver_stream.date_created < %%(time_end)s
|
||||
GROUP BY zerver_realm.id %(group_by_clause)s
|
||||
"""
|
||||
|
||||
## CountStat declarations ##
|
||||
|
||||
count_stats_ = [
|
||||
# Messages Sent stats
|
||||
# Stats that count the number of messages sent in various ways.
|
||||
# These are also the set of stats that read from the Message table.
|
||||
|
||||
CountStat('messages_sent:is_bot:hour',
|
||||
sql_data_collector(UserCount, count_message_by_user_query, (UserProfile, 'is_bot')),
|
||||
CountStat.HOUR),
|
||||
CountStat('messages_sent:message_type:day',
|
||||
sql_data_collector(UserCount, count_message_type_by_user_query, None), CountStat.DAY),
|
||||
CountStat('messages_sent:client:day',
|
||||
sql_data_collector(UserCount, count_message_by_user_query, (Message, 'sending_client_id')),
|
||||
CountStat.DAY),
|
||||
CountStat('messages_in_stream:is_bot:day',
|
||||
sql_data_collector(StreamCount, count_message_by_stream_query, (UserProfile, 'is_bot')),
|
||||
CountStat.DAY),
|
||||
|
||||
# Number of Users stats
|
||||
# Stats that count the number of active users in the UserProfile.is_active sense.
|
||||
|
||||
# 'active_users_audit:is_bot:day' is the canonical record of which users were
|
||||
# active on which days (in the UserProfile.is_active sense).
|
||||
# Important that this stay a daily stat, so that 'realm_active_humans::day' works as expected.
|
||||
CountStat('active_users_audit:is_bot:day',
|
||||
sql_data_collector(UserCount, check_realmauditlog_by_user_query, (UserProfile, 'is_bot')),
|
||||
CountStat.DAY),
|
||||
# Sanity check on 'active_users_audit:is_bot:day', and a archetype for future LoggingCountStats.
|
||||
# In RealmCount, 'active_users_audit:is_bot:day' should be the partial
|
||||
# sum sequence of 'active_users_log:is_bot:day', for any realm that
|
||||
# started after the latter stat was introduced.
|
||||
LoggingCountStat('active_users_log:is_bot:day', RealmCount, CountStat.DAY),
|
||||
# Another sanity check on 'active_users_audit:is_bot:day'. Is only an
|
||||
# approximation, e.g. if a user is deactivated between the end of the
|
||||
# day and when this stat is run, they won't be counted. However, is the
|
||||
# simplest of the three to inspect by hand.
|
||||
CountStat('active_users:is_bot:day',
|
||||
sql_data_collector(RealmCount, count_user_by_realm_query, (UserProfile, 'is_bot')),
|
||||
CountStat.DAY, interval=TIMEDELTA_MAX),
|
||||
|
||||
# User Activity stats
|
||||
# Stats that measure user activity in the UserActivityInterval sense.
|
||||
|
||||
CountStat('1day_actives::day',
|
||||
sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
|
||||
CountStat.DAY, interval=timedelta(days=1)-UserActivityInterval.MIN_INTERVAL_LENGTH),
|
||||
CountStat('15day_actives::day',
|
||||
sql_data_collector(UserCount, check_useractivityinterval_by_user_query, None),
|
||||
CountStat.DAY, interval=timedelta(days=15)-UserActivityInterval.MIN_INTERVAL_LENGTH),
|
||||
CountStat('minutes_active::day', DataCollector(UserCount, do_pull_minutes_active), CountStat.DAY),
|
||||
|
||||
# Rate limiting stats
|
||||
|
||||
# Used to limit the number of invitation emails sent by a realm
|
||||
LoggingCountStat('invites_sent::day', RealmCount, CountStat.DAY),
|
||||
|
||||
# Dependent stats
|
||||
# Must come after their dependencies.
|
||||
|
||||
# Canonical account of the number of active humans in a realm on each day.
|
||||
DependentCountStat('realm_active_humans::day',
|
||||
sql_data_collector(RealmCount, count_realm_active_humans_query, None),
|
||||
CountStat.DAY,
|
||||
dependencies=['active_users_audit:is_bot:day', '15day_actives::day'])
|
||||
]
|
||||
|
||||
COUNT_STATS = OrderedDict([(stat.property, stat) for stat in count_stats_])
|
||||
@@ -1,63 +0,0 @@
|
||||
from math import sqrt
|
||||
from random import gauss, random, seed
|
||||
from typing import List
|
||||
|
||||
from analytics.lib.counts import CountStat
|
||||
|
||||
def generate_time_series_data(days: int=100, business_hours_base: float=10,
|
||||
non_business_hours_base: float=10, growth: float=1,
|
||||
autocorrelation: float=0, spikiness: float=1,
|
||||
holiday_rate: float=0, frequency: str=CountStat.DAY,
|
||||
partial_sum: bool=False, random_seed: int=26) -> List[int]:
|
||||
"""
|
||||
Generate semi-realistic looking time series data for testing analytics graphs.
|
||||
|
||||
days -- Number of days of data. Is the number of data points generated if
|
||||
frequency is CountStat.DAY.
|
||||
business_hours_base -- Average value during a business hour (or day) at beginning of
|
||||
time series, if frequency is CountStat.HOUR (CountStat.DAY, respectively).
|
||||
non_business_hours_base -- The above, for non-business hours/days.
|
||||
growth -- Ratio between average values at end of time series and beginning of time series.
|
||||
autocorrelation -- Makes neighboring data points look more like each other. At 0 each
|
||||
point is unaffected by the previous point, and at 1 each point is a deterministic
|
||||
function of the previous point.
|
||||
spikiness -- 0 means no randomness (other than holiday_rate), higher values increase
|
||||
the variance.
|
||||
holiday_rate -- Fraction of days randomly set to 0, largely for testing how we handle 0s.
|
||||
frequency -- Should be CountStat.HOUR or CountStat.DAY.
|
||||
partial_sum -- If True, return partial sum of the series.
|
||||
random_seed -- Seed for random number generator.
|
||||
"""
|
||||
if frequency == CountStat.HOUR:
|
||||
length = days*24
|
||||
seasonality = [non_business_hours_base] * 24 * 7
|
||||
for day in range(5):
|
||||
for hour in range(8):
|
||||
seasonality[24*day + hour] = business_hours_base
|
||||
holidays = []
|
||||
for i in range(days):
|
||||
holidays.extend([random() < holiday_rate] * 24)
|
||||
elif frequency == CountStat.DAY:
|
||||
length = days
|
||||
seasonality = [8*business_hours_base + 16*non_business_hours_base] * 5 + \
|
||||
[24*non_business_hours_base] * 2
|
||||
holidays = [random() < holiday_rate for i in range(days)]
|
||||
else:
|
||||
raise AssertionError("Unknown frequency: %s" % (frequency,))
|
||||
if length < 2:
|
||||
raise AssertionError("Must be generating at least 2 data points. "
|
||||
"Currently generating %s" % (length,))
|
||||
growth_base = growth ** (1. / (length-1))
|
||||
values_no_noise = [seasonality[i % len(seasonality)] * (growth_base**i) for i in range(length)]
|
||||
|
||||
seed(random_seed)
|
||||
noise_scalars = [gauss(0, 1)]
|
||||
for i in range(1, length):
|
||||
noise_scalars.append(noise_scalars[-1]*autocorrelation + gauss(0, 1)*(1-autocorrelation))
|
||||
|
||||
values = [0 if holiday else int(v + sqrt(v)*noise_scalar*spikiness)
|
||||
for v, noise_scalar, holiday in zip(values_no_noise, noise_scalars, holidays)]
|
||||
if partial_sum:
|
||||
for i in range(1, length):
|
||||
values[i] = values[i-1] + values[i]
|
||||
return [max(v, 0) for v in values]
|
||||
@@ -1,31 +0,0 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Optional
|
||||
|
||||
from analytics.lib.counts import CountStat
|
||||
from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC
|
||||
|
||||
# If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive.
|
||||
# If min_length is greater than 0, pads the list to the left.
|
||||
# So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22],
|
||||
# and time_range(Sep 20, Sep 22, day, 5) returns [Sep 18, Sep 19, Sep 20, Sep 21, Sep 22]
|
||||
def time_range(start: datetime, end: datetime, frequency: str,
|
||||
min_length: Optional[int]) -> List[datetime]:
|
||||
verify_UTC(start)
|
||||
verify_UTC(end)
|
||||
if frequency == CountStat.HOUR:
|
||||
end = floor_to_hour(end)
|
||||
step = timedelta(hours=1)
|
||||
elif frequency == CountStat.DAY:
|
||||
end = floor_to_day(end)
|
||||
step = timedelta(days=1)
|
||||
else:
|
||||
raise AssertionError("Unknown frequency: %s" % (frequency,))
|
||||
|
||||
times = []
|
||||
if min_length is not None:
|
||||
start = min(start, end - (min_length-1)*step)
|
||||
current = end
|
||||
while current >= start:
|
||||
times.append(current)
|
||||
current -= step
|
||||
return list(reversed(times))
|
||||
57
analytics/management/commands/active_user_stats.py
Normal file
57
analytics/management/commands/active_user_stats.py
Normal file
@@ -0,0 +1,57 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from zerver.models import UserPresence, UserActivity
|
||||
from zerver.lib.utils import statsd, statsd_key
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Sends active user statistics to statsd.
|
||||
|
||||
Run as a cron job that runs every 10 minutes."""
|
||||
|
||||
def handle(self, *args, **options):
|
||||
# Get list of all active users in the last 1 week
|
||||
cutoff = datetime.now() - timedelta(minutes=30, hours=168)
|
||||
|
||||
users = UserPresence.objects.select_related().filter(timestamp__gt=cutoff)
|
||||
|
||||
# Calculate 10min, 2hrs, 12hrs, 1day, 2 business days (TODO business days), 1 week bucket of stats
|
||||
hour_buckets = [0.16, 2, 12, 24, 48, 168]
|
||||
user_info = defaultdict(dict)
|
||||
|
||||
for last_presence in users:
|
||||
if last_presence.status == UserPresence.IDLE:
|
||||
known_active = last_presence.timestamp - timedelta(minutes=30)
|
||||
else:
|
||||
known_active = last_presence.timestamp
|
||||
|
||||
for bucket in hour_buckets:
|
||||
if not bucket in user_info[last_presence.user_profile.realm.domain]:
|
||||
user_info[last_presence.user_profile.realm.domain][bucket] = []
|
||||
if datetime.now(known_active.tzinfo) - known_active < timedelta(hours=bucket):
|
||||
user_info[last_presence.user_profile.realm.domain][bucket].append(last_presence.user_profile.email)
|
||||
|
||||
for realm, buckets in user_info.items():
|
||||
print("Realm %s" % realm)
|
||||
for hr, users in sorted(buckets.items()):
|
||||
print("\tUsers for %s: %s" % (hr, len(users)))
|
||||
statsd.gauge("users.active.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
|
||||
|
||||
# Also do stats for how many users have been reading the app.
|
||||
users_reading = UserActivity.objects.select_related().filter(query="/json/update_message_flags")
|
||||
user_info = defaultdict(dict)
|
||||
for activity in users_reading:
|
||||
for bucket in hour_buckets:
|
||||
if not bucket in user_info[activity.user_profile.realm.domain]:
|
||||
user_info[activity.user_profile.realm.domain][bucket] = []
|
||||
if datetime.now(activity.last_visit.tzinfo) - activity.last_visit < timedelta(hours=bucket):
|
||||
user_info[activity.user_profile.realm.domain][bucket].append(activity.user_profile.email)
|
||||
for realm, buckets in user_info.items():
|
||||
print("Realm %s" % realm)
|
||||
for hr, users in sorted(buckets.items()):
|
||||
print("\tUsers reading for %s: %s" % (hr, len(users)))
|
||||
statsd.gauge("users.reading.%s.%shr" % (statsd_key(realm, True), statsd_key(hr, True)), len(users))
|
||||
24
analytics/management/commands/active_user_stats_by_day.py
Normal file
24
analytics/management/commands/active_user_stats_by_day.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
from optparse import make_option
|
||||
from django.core.management.base import BaseCommand
|
||||
from zerver.lib.statistics import activity_averages_during_day
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Generate statistics on user activity for a given day."
|
||||
|
||||
option_list = BaseCommand.option_list + \
|
||||
(make_option('--date', default=None, action='store',
|
||||
help="Day to query in format 2013-12-05. Default is yesterday"),)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options["date"] is None:
|
||||
date = datetime.datetime.now() - datetime.timedelta(days=1)
|
||||
else:
|
||||
date = datetime.datetime.strptime(options["date"], "%Y-%m-%d")
|
||||
print "Activity data for", date
|
||||
print activity_averages_during_day(date)
|
||||
print "Please note that the total registered user count is a total for today"
|
||||
@@ -1,21 +1,21 @@
|
||||
import datetime
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandParser
|
||||
from __future__ import absolute_import
|
||||
|
||||
from optparse import make_option
|
||||
from django.core.management.base import BaseCommand
|
||||
from zerver.models import Recipient, Message
|
||||
from zerver.lib.timestamp import timestamp_to_datetime
|
||||
from zerver.models import Message, Recipient
|
||||
import datetime
|
||||
import time
|
||||
import logging
|
||||
|
||||
def compute_stats(log_level: int) -> None:
|
||||
def compute_stats(log_level):
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(log_level)
|
||||
|
||||
one_week_ago = timestamp_to_datetime(time.time()) - datetime.timedelta(weeks=1)
|
||||
mit_query = Message.objects.filter(sender__realm__string_id="zephyr",
|
||||
mit_query = Message.objects.filter(sender__realm__domain="mit.edu",
|
||||
recipient__type=Recipient.STREAM,
|
||||
date_sent__gt=one_week_ago)
|
||||
pub_date__gt=one_week_ago)
|
||||
for bot_sender_start in ["imap.", "rcmd.", "sys."]:
|
||||
mit_query = mit_query.exclude(sender__email__startswith=(bot_sender_start))
|
||||
# Filtering for "/" covers tabbott/extra@ and all the daemon/foo bots.
|
||||
@@ -26,15 +26,15 @@ def compute_stats(log_level: int) -> None:
|
||||
"bitcoin@mit.edu", "lp@mit.edu", "clocks@mit.edu",
|
||||
"root@mit.edu", "nagios@mit.edu",
|
||||
"www-data|local-realm@mit.edu"])
|
||||
user_counts = {} # type: Dict[str, Dict[str, int]]
|
||||
user_counts = {}
|
||||
for m in mit_query.select_related("sending_client", "sender"):
|
||||
email = m.sender.email
|
||||
user_counts.setdefault(email, {})
|
||||
user_counts[email].setdefault(m.sending_client.name, 0)
|
||||
user_counts[email][m.sending_client.name] += 1
|
||||
|
||||
total_counts = {} # type: Dict[str, int]
|
||||
total_user_counts = {} # type: Dict[str, int]
|
||||
total_counts = {}
|
||||
total_user_counts = {}
|
||||
for email, counts in user_counts.items():
|
||||
total_user_counts.setdefault(email, 0)
|
||||
for client_name, count in counts.items():
|
||||
@@ -43,13 +43,13 @@ def compute_stats(log_level: int) -> None:
|
||||
total_user_counts[email] += count
|
||||
|
||||
logging.debug("%40s | %10s | %s" % ("User", "Messages", "Percentage Zulip"))
|
||||
top_percents = {} # type: Dict[int, float]
|
||||
top_percents = {}
|
||||
for size in [10, 25, 50, 100, 200, len(total_user_counts.keys())]:
|
||||
top_percents[size] = 0.0
|
||||
top_percents[size] = 0
|
||||
for i, email in enumerate(sorted(total_user_counts.keys(),
|
||||
key=lambda x: -total_user_counts[x])):
|
||||
percent_zulip = round(100 - (user_counts[email].get("zephyr_mirror", 0)) * 100. /
|
||||
total_user_counts[email], 1)
|
||||
total_user_counts[email], 1)
|
||||
for size in top_percents.keys():
|
||||
top_percents.setdefault(size, 0)
|
||||
if i < size:
|
||||
@@ -63,18 +63,18 @@ def compute_stats(log_level: int) -> None:
|
||||
logging.info("Top %6s | %s%%" % (size, round(top_percents[size], 1)))
|
||||
|
||||
grand_total = sum(total_counts.values())
|
||||
print(grand_total)
|
||||
print grand_total
|
||||
logging.info("%15s | %s" % ("Client", "Percentage"))
|
||||
for client in total_counts.keys():
|
||||
logging.info("%15s | %s%%" % (client, round(100. * total_counts[client] / grand_total, 1)))
|
||||
|
||||
class Command(BaseCommand):
|
||||
option_list = BaseCommand.option_list + \
|
||||
(make_option('--verbose', default=False, action='store_true'),)
|
||||
|
||||
help = "Compute statistics on MIT Zephyr usage."
|
||||
|
||||
def add_arguments(self, parser: CommandParser) -> None:
|
||||
parser.add_argument('--verbose', default=False, action='store_true')
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
def handle(self, *args, **options):
|
||||
level = logging.INFO
|
||||
if options["verbose"]:
|
||||
level = logging.DEBUG
|
||||
|
||||
@@ -1,21 +1,22 @@
|
||||
import datetime
|
||||
from typing import Any, Dict
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandParser
|
||||
from django.utils.timezone import utc
|
||||
from __future__ import absolute_import
|
||||
|
||||
from zerver.lib.statistics import seconds_usage_between
|
||||
from zerver.models import UserProfile
|
||||
|
||||
def analyze_activity(options: Dict[str, Any]) -> None:
|
||||
from optparse import make_option
|
||||
from django.core.management.base import BaseCommand
|
||||
from zerver.models import UserProfile
|
||||
import datetime
|
||||
from django.utils.timezone import utc
|
||||
|
||||
def analyze_activity(options):
|
||||
day_start = datetime.datetime.strptime(options["date"], "%Y-%m-%d").replace(tzinfo=utc)
|
||||
day_end = day_start + datetime.timedelta(days=options["duration"])
|
||||
|
||||
user_profile_query = UserProfile.objects.all()
|
||||
if options["realm"]:
|
||||
user_profile_query = user_profile_query.filter(realm__string_id=options["realm"])
|
||||
user_profile_query = user_profile_query.filter(realm__domain=options["realm"])
|
||||
|
||||
print("Per-user online duration:\n")
|
||||
print "Per-user online duration:\n"
|
||||
total_duration = datetime.timedelta(0)
|
||||
for user_profile in user_profile_query:
|
||||
duration = seconds_usage_between(user_profile, day_start, day_end)
|
||||
@@ -24,11 +25,11 @@ def analyze_activity(options: Dict[str, Any]) -> None:
|
||||
continue
|
||||
|
||||
total_duration += duration
|
||||
print("%-*s%s" % (37, user_profile.email, duration,))
|
||||
print "%-*s%s" % (37, user_profile.email, duration, )
|
||||
|
||||
print("\nTotal Duration: %s" % (total_duration,))
|
||||
print("\nTotal Duration in minutes: %s" % (total_duration.total_seconds() / 60.,))
|
||||
print("Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,))
|
||||
print "\nTotal Duration: %s" % (total_duration,)
|
||||
print "\nTotal Duration in minutes: %s" % (total_duration.total_seconds() / 60.,)
|
||||
print "Total Duration amortized to a month: %s" % (total_duration.total_seconds() * 30. / 60.,)
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Report analytics of user activity on a per-user and realm basis.
|
||||
@@ -41,16 +42,16 @@ It will correctly not count server-initiated reloads in the activity statistics.
|
||||
|
||||
The duration flag can be used to control how many days to show usage duration for
|
||||
|
||||
Usage: ./manage.py analyze_user_activity [--realm=zulip] [--date=2013-09-10] [--duration=1]
|
||||
Usage: python manage.py analyze_user_activity [--realm=zulip.com] [--date=2013-09-10] [--duration=1]
|
||||
|
||||
By default, if no date is selected 2013-09-10 is used. If no realm is provided, information
|
||||
is shown for all realms"""
|
||||
|
||||
def add_arguments(self, parser: CommandParser) -> None:
|
||||
parser.add_argument('--realm', action='store')
|
||||
parser.add_argument('--date', action='store', default="2013-09-06")
|
||||
parser.add_argument('--duration', action='store', default=1, type=int,
|
||||
help="How many days to show usage information for")
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--realm', action='store'),
|
||||
make_option('--date', action='store', default="2013-09-06"),
|
||||
make_option('--duration', action='store', default=1, type=int, help="How many days to show usage information for"),
|
||||
)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
def handle(self, *args, **options):
|
||||
analyze_activity(options)
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils.timezone import now as timezone_now
|
||||
|
||||
from analytics.models import installation_epoch, \
|
||||
last_successful_fill
|
||||
from analytics.lib.counts import COUNT_STATS, CountStat
|
||||
from zerver.lib.timestamp import floor_to_hour, floor_to_day, verify_UTC, \
|
||||
TimezoneNotUTCException
|
||||
from zerver.models import Realm
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict
|
||||
|
||||
states = {
|
||||
0: "OK",
|
||||
1: "WARNING",
|
||||
2: "CRITICAL",
|
||||
3: "UNKNOWN"
|
||||
}
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Checks FillState table.
|
||||
|
||||
Run as a cron job that runs every hour."""
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
fill_state = self.get_fill_state()
|
||||
status = fill_state['status']
|
||||
message = fill_state['message']
|
||||
|
||||
state_file_path = "/var/lib/nagios_state/check-analytics-state"
|
||||
state_file_tmp = state_file_path + "-tmp"
|
||||
|
||||
with open(state_file_tmp, "w") as f:
|
||||
f.write("%s|%s|%s|%s\n" % (
|
||||
int(time.time()), status, states[status], message))
|
||||
os.rename(state_file_tmp, state_file_path)
|
||||
|
||||
def get_fill_state(self) -> Dict[str, Any]:
|
||||
if not Realm.objects.exists():
|
||||
return {'status': 0, 'message': 'No realms exist, so not checking FillState.'}
|
||||
|
||||
warning_unfilled_properties = []
|
||||
critical_unfilled_properties = []
|
||||
for property, stat in COUNT_STATS.items():
|
||||
last_fill = last_successful_fill(property)
|
||||
if last_fill is None:
|
||||
last_fill = installation_epoch()
|
||||
try:
|
||||
verify_UTC(last_fill)
|
||||
except TimezoneNotUTCException:
|
||||
return {'status': 2, 'message': 'FillState not in UTC for %s' % (property,)}
|
||||
|
||||
if stat.frequency == CountStat.DAY:
|
||||
floor_function = floor_to_day
|
||||
warning_threshold = timedelta(hours=26)
|
||||
critical_threshold = timedelta(hours=50)
|
||||
else: # CountStat.HOUR
|
||||
floor_function = floor_to_hour
|
||||
warning_threshold = timedelta(minutes=90)
|
||||
critical_threshold = timedelta(minutes=150)
|
||||
|
||||
if floor_function(last_fill) != last_fill:
|
||||
return {'status': 2, 'message': 'FillState not on %s boundary for %s' %
|
||||
(stat.frequency, property)}
|
||||
|
||||
time_to_last_fill = timezone_now() - last_fill
|
||||
if time_to_last_fill > critical_threshold:
|
||||
critical_unfilled_properties.append(property)
|
||||
elif time_to_last_fill > warning_threshold:
|
||||
warning_unfilled_properties.append(property)
|
||||
|
||||
if len(critical_unfilled_properties) == 0 and len(warning_unfilled_properties) == 0:
|
||||
return {'status': 0, 'message': 'FillState looks fine.'}
|
||||
if len(critical_unfilled_properties) == 0:
|
||||
return {'status': 1, 'message': 'Missed filling %s once.' %
|
||||
(', '.join(warning_unfilled_properties),)}
|
||||
return {'status': 2, 'message': 'Missed filling %s once. Missed filling %s at least twice.' %
|
||||
(', '.join(warning_unfilled_properties), ', '.join(critical_unfilled_properties))}
|
||||
@@ -1,20 +0,0 @@
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from analytics.lib.counts import do_drop_all_analytics_tables
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Clear analytics tables."""
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('--force',
|
||||
action='store_true',
|
||||
help="Clear analytics tables.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
if options['force']:
|
||||
do_drop_all_analytics_tables()
|
||||
else:
|
||||
raise CommandError("Would delete all data from analytics tables (!); use --force to do so.")
|
||||
@@ -1,26 +0,0 @@
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from analytics.lib.counts import COUNT_STATS, do_drop_single_stat
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Clear analytics tables."""
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('--force',
|
||||
action='store_true',
|
||||
help="Actually do it.")
|
||||
parser.add_argument('--property',
|
||||
type=str,
|
||||
help="The property of the stat to be cleared.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
property = options['property']
|
||||
if property not in COUNT_STATS:
|
||||
raise CommandError("Invalid property: %s" % (property,))
|
||||
if not options['force']:
|
||||
raise CommandError("No action taken. Use --force.")
|
||||
|
||||
do_drop_single_stat(property)
|
||||
@@ -1,32 +1,23 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Count
|
||||
|
||||
from zerver.models import UserActivity, UserProfile, Realm, \
|
||||
get_realm, get_user_profile_by_email
|
||||
|
||||
import datetime
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any, Optional
|
||||
|
||||
from django.db.models import Count, QuerySet
|
||||
from django.utils.timezone import now as timezone_now
|
||||
|
||||
from zerver.lib.management import ZulipBaseCommand
|
||||
from zerver.models import UserActivity
|
||||
|
||||
class Command(ZulipBaseCommand):
|
||||
class Command(BaseCommand):
|
||||
help = """Report rough client activity globally, for a realm, or for a user
|
||||
|
||||
Usage examples:
|
||||
|
||||
./manage.py client_activity --target server
|
||||
./manage.py client_activity --target realm --realm zulip
|
||||
./manage.py client_activity --target user --user hamlet@zulip.com --realm zulip"""
|
||||
python manage.py client_activity
|
||||
python manage.py client_activity zulip.com
|
||||
python manage.py client_activity jesstess@zulip.com"""
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('--target', dest='target', required=True, type=str,
|
||||
help="'server' will calculate client activity of the entire server. "
|
||||
"'realm' will calculate client activity of realm. "
|
||||
"'user' will calculate client activity of the user.")
|
||||
parser.add_argument('--user', dest='user', type=str,
|
||||
help="The email address of the user you want to calculate activity.")
|
||||
self.add_realm_args(parser)
|
||||
|
||||
def compute_activity(self, user_activity_objects: QuerySet) -> None:
|
||||
def compute_activity(self, user_activity_objects):
|
||||
# Report data from the past week.
|
||||
#
|
||||
# This is a rough report of client activity because we inconsistently
|
||||
@@ -37,7 +28,7 @@ Usage examples:
|
||||
#
|
||||
# Importantly, this does NOT tell you anything about the relative
|
||||
# volumes of requests from clients.
|
||||
threshold = timezone_now() - datetime.timedelta(days=7)
|
||||
threshold = datetime.datetime.now() - datetime.timedelta(days=7)
|
||||
client_counts = user_activity_objects.filter(
|
||||
last_visit__gt=threshold).values("client__name").annotate(
|
||||
count=Count('client__name'))
|
||||
@@ -53,21 +44,26 @@ Usage examples:
|
||||
counts.sort()
|
||||
|
||||
for count in counts:
|
||||
print("%25s %15d" % (count[1], count[0]))
|
||||
print("Total:", total)
|
||||
print "%25s %15d" % (count[1], count[0])
|
||||
print "Total:", total
|
||||
|
||||
def handle(self, *args: Any, **options: Optional[str]) -> None:
|
||||
realm = self.get_realm(options)
|
||||
if options["user"] is None:
|
||||
if options["target"] == "server" and realm is None:
|
||||
# Report global activity.
|
||||
self.compute_activity(UserActivity.objects.all())
|
||||
elif options["target"] == "realm" and realm is not None:
|
||||
self.compute_activity(UserActivity.objects.filter(user_profile__realm=realm))
|
||||
else:
|
||||
self.print_help("./manage.py", "client_activity")
|
||||
elif options["target"] == "user":
|
||||
user_profile = self.get_user(options["user"], realm)
|
||||
self.compute_activity(UserActivity.objects.filter(user_profile=user_profile))
|
||||
else:
|
||||
self.print_help("./manage.py", "client_activity")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if len(args) == 0:
|
||||
# Report global activity.
|
||||
self.compute_activity(UserActivity.objects.all())
|
||||
elif len(args) == 1:
|
||||
try:
|
||||
# Report activity for a user.
|
||||
user_profile = get_user_profile_by_email(args[0])
|
||||
self.compute_activity(UserActivity.objects.filter(
|
||||
user_profile=user_profile))
|
||||
except UserProfile.DoesNotExist:
|
||||
try:
|
||||
# Report activity for a realm.
|
||||
realm = get_realm(args[0])
|
||||
self.compute_activity(UserActivity.objects.filter(
|
||||
user_profile__realm=realm))
|
||||
except Realm.DoesNotExist:
|
||||
print "Unknown user or domain %s" % (args[0],)
|
||||
exit(1)
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
from datetime import timedelta
|
||||
from typing import Any, Dict, List, Mapping, Optional, Type
|
||||
import mock
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils.timezone import now as timezone_now
|
||||
|
||||
from analytics.lib.counts import COUNT_STATS, \
|
||||
CountStat, do_drop_all_analytics_tables
|
||||
from analytics.lib.fixtures import generate_time_series_data
|
||||
from analytics.lib.time_utils import time_range
|
||||
from analytics.models import BaseCount, FillState, RealmCount, UserCount, \
|
||||
StreamCount, InstallationCount
|
||||
from zerver.lib.actions import do_change_is_admin, STREAM_ASSIGNMENT_COLORS
|
||||
from zerver.lib.create_user import create_user
|
||||
from zerver.lib.timestamp import floor_to_day
|
||||
from zerver.models import Realm, Stream, Client, \
|
||||
Recipient, Subscription
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Populates analytics tables with randomly generated data."""
|
||||
|
||||
DAYS_OF_DATA = 100
|
||||
random_seed = 26
|
||||
|
||||
def generate_fixture_data(self, stat: CountStat, business_hours_base: float,
|
||||
non_business_hours_base: float, growth: float,
|
||||
autocorrelation: float, spikiness: float,
|
||||
holiday_rate: float=0, partial_sum: bool=False) -> List[int]:
|
||||
self.random_seed += 1
|
||||
return generate_time_series_data(
|
||||
days=self.DAYS_OF_DATA, business_hours_base=business_hours_base,
|
||||
non_business_hours_base=non_business_hours_base, growth=growth,
|
||||
autocorrelation=autocorrelation, spikiness=spikiness, holiday_rate=holiday_rate,
|
||||
frequency=stat.frequency, partial_sum=partial_sum, random_seed=self.random_seed)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
# TODO: This should arguably only delete the objects
|
||||
# associated with the "analytics" realm.
|
||||
do_drop_all_analytics_tables()
|
||||
|
||||
# This also deletes any objects with this realm as a foreign key
|
||||
Realm.objects.filter(string_id='analytics').delete()
|
||||
|
||||
# Because we just deleted a bunch of objects in the database
|
||||
# directly (rather than deleting individual objects in Django,
|
||||
# in which case our post_save hooks would have flushed the
|
||||
# individual objects from memcached for us), we need to flush
|
||||
# memcached in order to ensure deleted objects aren't still
|
||||
# present in the memcached cache.
|
||||
from zerver.apps import flush_cache
|
||||
flush_cache(None)
|
||||
|
||||
installation_time = timezone_now() - timedelta(days=self.DAYS_OF_DATA)
|
||||
last_end_time = floor_to_day(timezone_now())
|
||||
realm = Realm.objects.create(
|
||||
string_id='analytics', name='Analytics', date_created=installation_time)
|
||||
with mock.patch("zerver.lib.create_user.timezone_now", return_value=installation_time):
|
||||
shylock = create_user('shylock@analytics.ds', 'Shylock', realm,
|
||||
full_name='Shylock', short_name='shylock',
|
||||
is_realm_admin=True)
|
||||
do_change_is_admin(shylock, True)
|
||||
stream = Stream.objects.create(
|
||||
name='all', realm=realm, date_created=installation_time)
|
||||
recipient = Recipient.objects.create(type_id=stream.id, type=Recipient.STREAM)
|
||||
stream.recipient = recipient
|
||||
stream.save(update_fields=["recipient"])
|
||||
|
||||
# Subscribe shylock to the stream to avoid invariant failures.
|
||||
# TODO: This should use subscribe_users_to_streams from populate_db.
|
||||
subs = [
|
||||
Subscription(recipient=recipient,
|
||||
user_profile=shylock,
|
||||
color=STREAM_ASSIGNMENT_COLORS[0]),
|
||||
]
|
||||
Subscription.objects.bulk_create(subs)
|
||||
|
||||
def insert_fixture_data(stat: CountStat,
|
||||
fixture_data: Mapping[Optional[str], List[int]],
|
||||
table: Type[BaseCount]) -> None:
|
||||
end_times = time_range(last_end_time, last_end_time, stat.frequency,
|
||||
len(list(fixture_data.values())[0]))
|
||||
if table == InstallationCount:
|
||||
id_args = {} # type: Dict[str, Any]
|
||||
if table == RealmCount:
|
||||
id_args = {'realm': realm}
|
||||
if table == UserCount:
|
||||
id_args = {'realm': realm, 'user': shylock}
|
||||
if table == StreamCount:
|
||||
id_args = {'stream': stream, 'realm': realm}
|
||||
|
||||
for subgroup, values in fixture_data.items():
|
||||
table.objects.bulk_create([
|
||||
table(property=stat.property, subgroup=subgroup, end_time=end_time,
|
||||
value=value, **id_args)
|
||||
for end_time, value in zip(end_times, values) if value != 0])
|
||||
|
||||
stat = COUNT_STATS['1day_actives::day']
|
||||
realm_data = {
|
||||
None: self.generate_fixture_data(stat, .08, .02, 3, .3, 6, partial_sum=True),
|
||||
} # type: Mapping[Optional[str], List[int]]
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
None: self.generate_fixture_data(stat, .8, .2, 4, .3, 6, partial_sum=True),
|
||||
} # type: Mapping[Optional[str], List[int]]
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
realm_data = {
|
||||
None: self.generate_fixture_data(stat, .1, .03, 3, .5, 3, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
None: self.generate_fixture_data(stat, 1, .3, 4, .5, 3, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
stat = COUNT_STATS['active_users_audit:is_bot:day']
|
||||
realm_data = {
|
||||
'false': self.generate_fixture_data(stat, .1, .03, 3.5, .8, 2, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
'false': self.generate_fixture_data(stat, 1, .3, 6, .8, 2, partial_sum=True),
|
||||
}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
stat = COUNT_STATS['messages_sent:is_bot:hour']
|
||||
user_data = {'false': self.generate_fixture_data(
|
||||
stat, 2, 1, 1.5, .6, 8, holiday_rate=.1)} # type: Mapping[Optional[str], List[int]]
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {'false': self.generate_fixture_data(stat, 35, 15, 6, .6, 4),
|
||||
'true': self.generate_fixture_data(stat, 15, 15, 3, .4, 2)}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {'false': self.generate_fixture_data(stat, 350, 150, 6, .6, 4),
|
||||
'true': self.generate_fixture_data(stat, 150, 150, 3, .4, 2)}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
stat = COUNT_STATS['messages_sent:message_type:day']
|
||||
user_data = {
|
||||
'public_stream': self.generate_fixture_data(stat, 1.5, 1, 3, .6, 8),
|
||||
'private_message': self.generate_fixture_data(stat, .5, .3, 1, .6, 8),
|
||||
'huddle_message': self.generate_fixture_data(stat, .2, .2, 2, .6, 8)}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {
|
||||
'public_stream': self.generate_fixture_data(stat, 30, 8, 5, .6, 4),
|
||||
'private_stream': self.generate_fixture_data(stat, 7, 7, 5, .6, 4),
|
||||
'private_message': self.generate_fixture_data(stat, 13, 5, 5, .6, 4),
|
||||
'huddle_message': self.generate_fixture_data(stat, 6, 3, 3, .6, 4)}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
'public_stream': self.generate_fixture_data(stat, 300, 80, 5, .6, 4),
|
||||
'private_stream': self.generate_fixture_data(stat, 70, 70, 5, .6, 4),
|
||||
'private_message': self.generate_fixture_data(stat, 130, 50, 5, .6, 4),
|
||||
'huddle_message': self.generate_fixture_data(stat, 60, 30, 3, .6, 4)}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
website, created = Client.objects.get_or_create(name='website')
|
||||
old_desktop, created = Client.objects.get_or_create(name='desktop app Linux 0.3.7')
|
||||
android, created = Client.objects.get_or_create(name='ZulipAndroid')
|
||||
iOS, created = Client.objects.get_or_create(name='ZulipiOS')
|
||||
react_native, created = Client.objects.get_or_create(name='ZulipMobile')
|
||||
API, created = Client.objects.get_or_create(name='API: Python')
|
||||
zephyr_mirror, created = Client.objects.get_or_create(name='zephyr_mirror')
|
||||
unused, created = Client.objects.get_or_create(name='unused')
|
||||
long_webhook, created = Client.objects.get_or_create(name='ZulipLooooooooooongNameWebhook')
|
||||
|
||||
stat = COUNT_STATS['messages_sent:client:day']
|
||||
user_data = {
|
||||
website.id: self.generate_fixture_data(stat, 2, 1, 1.5, .6, 8),
|
||||
zephyr_mirror.id: self.generate_fixture_data(stat, 0, .3, 1.5, .6, 8)}
|
||||
insert_fixture_data(stat, user_data, UserCount)
|
||||
realm_data = {
|
||||
website.id: self.generate_fixture_data(stat, 30, 20, 5, .6, 3),
|
||||
old_desktop.id: self.generate_fixture_data(stat, 5, 3, 8, .6, 3),
|
||||
android.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
|
||||
iOS.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3),
|
||||
react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
|
||||
API.id: self.generate_fixture_data(stat, 5, 5, 5, .6, 3),
|
||||
zephyr_mirror.id: self.generate_fixture_data(stat, 1, 1, 3, .6, 3),
|
||||
unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
|
||||
long_webhook.id: self.generate_fixture_data(stat, 5, 5, 2, .6, 3)}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
installation_data = {
|
||||
website.id: self.generate_fixture_data(stat, 300, 200, 5, .6, 3),
|
||||
old_desktop.id: self.generate_fixture_data(stat, 50, 30, 8, .6, 3),
|
||||
android.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
|
||||
iOS.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3),
|
||||
react_native.id: self.generate_fixture_data(stat, 5, 5, 10, .6, 3),
|
||||
API.id: self.generate_fixture_data(stat, 50, 50, 5, .6, 3),
|
||||
zephyr_mirror.id: self.generate_fixture_data(stat, 10, 10, 3, .6, 3),
|
||||
unused.id: self.generate_fixture_data(stat, 0, 0, 0, 0, 0),
|
||||
long_webhook.id: self.generate_fixture_data(stat, 50, 50, 2, .6, 3)}
|
||||
insert_fixture_data(stat, installation_data, InstallationCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
|
||||
stat = COUNT_STATS['messages_in_stream:is_bot:day']
|
||||
realm_data = {'false': self.generate_fixture_data(stat, 30, 5, 6, .6, 4),
|
||||
'true': self.generate_fixture_data(stat, 20, 2, 3, .2, 3)}
|
||||
insert_fixture_data(stat, realm_data, RealmCount)
|
||||
stream_data = {'false': self.generate_fixture_data(stat, 10, 7, 5, .6, 4),
|
||||
'true': self.generate_fixture_data(stat, 5, 3, 2, .4, 2)} # type: Mapping[Optional[str], List[int]]
|
||||
insert_fixture_data(stat, stream_data, StreamCount)
|
||||
FillState.objects.create(property=stat.property, end_time=last_end_time,
|
||||
state=FillState.DONE)
|
||||
@@ -1,13 +1,12 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import datetime
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any, List
|
||||
import pytz
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Count
|
||||
from django.utils.timezone import now as timezone_now
|
||||
|
||||
from zerver.models import Message, Realm, Recipient, Stream, \
|
||||
Subscription, UserActivity, UserMessage, UserProfile, get_realm
|
||||
from zerver.models import UserProfile, Realm, Stream, Message, Recipient, UserActivity, \
|
||||
Subscription, UserMessage
|
||||
|
||||
MOBILE_CLIENT_LIST = ["Android", "ios"]
|
||||
HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"]
|
||||
@@ -17,99 +16,97 @@ human_messages = Message.objects.filter(sending_client__name__in=HUMAN_CLIENT_LI
|
||||
class Command(BaseCommand):
|
||||
help = "Generate statistics on realm activity."
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
|
||||
help="realm to generate statistics for")
|
||||
|
||||
def active_users(self, realm: Realm) -> List[UserProfile]:
|
||||
def active_users(self, realm):
|
||||
# Has been active (on the website, for now) in the last 7 days.
|
||||
activity_cutoff = timezone_now() - datetime.timedelta(days=7)
|
||||
return [activity.user_profile for activity in (
|
||||
UserActivity.objects.filter(user_profile__realm=realm,
|
||||
user_profile__is_active=True,
|
||||
last_visit__gt=activity_cutoff,
|
||||
query="/json/users/me/pointer",
|
||||
client__name="website"))]
|
||||
activity_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=7)
|
||||
return [activity.user_profile for activity in \
|
||||
UserActivity.objects.filter(user_profile__realm=realm,
|
||||
user_profile__is_active=True,
|
||||
last_visit__gt=activity_cutoff,
|
||||
query="/json/update_pointer",
|
||||
client__name="website")]
|
||||
|
||||
def messages_sent_by(self, user: UserProfile, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender=user, date_sent__gt=sent_time_cutoff).count()
|
||||
def messages_sent_by(self, user, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender=user, pub_date__gt=sent_time_cutoff).count()
|
||||
|
||||
def total_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return Message.objects.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
|
||||
def total_messages(self, realm, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return Message.objects.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
|
||||
|
||||
def human_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).count()
|
||||
def human_messages(self, realm, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).count()
|
||||
|
||||
def api_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
def api_messages(self, realm, days_ago):
|
||||
return (self.total_messages(realm, days_ago) - self.human_messages(realm, days_ago))
|
||||
|
||||
def stream_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff,
|
||||
def stream_messages(self, realm, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff,
|
||||
recipient__type=Recipient.STREAM).count()
|
||||
|
||||
def private_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
|
||||
def private_messages(self, realm, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
|
||||
recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.HUDDLE).count()
|
||||
|
||||
def group_private_messages(self, realm: Realm, days_ago: int) -> int:
|
||||
sent_time_cutoff = timezone_now() - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, date_sent__gt=sent_time_cutoff).exclude(
|
||||
def group_private_messages(self, realm, days_ago):
|
||||
sent_time_cutoff = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=days_ago)
|
||||
return human_messages.filter(sender__realm=realm, pub_date__gt=sent_time_cutoff).exclude(
|
||||
recipient__type=Recipient.STREAM).exclude(recipient__type=Recipient.PERSONAL).count()
|
||||
|
||||
def report_percentage(self, numerator: float, denominator: float, text: str) -> None:
|
||||
def report_percentage(self, numerator, denominator, text):
|
||||
if not denominator:
|
||||
fraction = 0.0
|
||||
else:
|
||||
fraction = numerator / float(denominator)
|
||||
print("%.2f%% of" % (fraction * 100,), text)
|
||||
print "%.2f%% of" % (fraction * 100,), text
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
if options['realms']:
|
||||
def handle(self, *args, **options):
|
||||
if args:
|
||||
try:
|
||||
realms = [get_realm(string_id) for string_id in options['realms']]
|
||||
except Realm.DoesNotExist as e:
|
||||
raise CommandError(e)
|
||||
realms = [Realm.objects.get(domain=domain) for domain in args]
|
||||
except Realm.DoesNotExist, e:
|
||||
print e
|
||||
exit(1)
|
||||
else:
|
||||
realms = Realm.objects.all()
|
||||
|
||||
for realm in realms:
|
||||
print(realm.string_id)
|
||||
print realm.domain
|
||||
|
||||
user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
|
||||
active_users = self.active_users(realm)
|
||||
num_active = len(active_users)
|
||||
|
||||
print("%d active users (%d total)" % (num_active, len(user_profiles)))
|
||||
print "%d active users (%d total)" % (num_active, len(user_profiles))
|
||||
streams = Stream.objects.filter(realm=realm).extra(
|
||||
tables=['zerver_subscription', 'zerver_recipient'],
|
||||
where=['zerver_subscription.recipient_id = zerver_recipient.id',
|
||||
'zerver_recipient.type = 2',
|
||||
'zerver_recipient.type_id = zerver_stream.id',
|
||||
'zerver_subscription.active = true']).annotate(count=Count("name"))
|
||||
print("%d streams" % (streams.count(),))
|
||||
print "%d streams" % (streams.count(),)
|
||||
|
||||
for days_ago in (1, 7, 30):
|
||||
print("In last %d days, users sent:" % (days_ago,))
|
||||
print "In last %d days, users sent:" % (days_ago,)
|
||||
sender_quantities = [self.messages_sent_by(user, days_ago) for user in user_profiles]
|
||||
for quantity in sorted(sender_quantities, reverse=True):
|
||||
print(quantity, end=' ')
|
||||
print("")
|
||||
print quantity,
|
||||
print ""
|
||||
|
||||
print("%d stream messages" % (self.stream_messages(realm, days_ago),))
|
||||
print("%d one-on-one private messages" % (self.private_messages(realm, days_ago),))
|
||||
print("%d messages sent via the API" % (self.api_messages(realm, days_ago),))
|
||||
print("%d group private messages" % (self.group_private_messages(realm, days_ago),))
|
||||
print "%d stream messages" % (self.stream_messages(realm, days_ago),)
|
||||
print "%d one-on-one private messages" % (self.private_messages(realm, days_ago),)
|
||||
print "%d messages sent via the API" % (self.api_messages(realm, days_ago),)
|
||||
print "%d group private messages" % (self.group_private_messages(realm, days_ago),)
|
||||
|
||||
num_notifications_enabled = len([x for x in active_users if x.enable_desktop_notifications])
|
||||
num_notifications_enabled = len(filter(lambda x: x.enable_desktop_notifications == True,
|
||||
active_users))
|
||||
self.report_percentage(num_notifications_enabled, num_active,
|
||||
"active users have desktop notifications enabled")
|
||||
|
||||
num_enter_sends = len([x for x in active_users if x.enter_sends])
|
||||
num_enter_sends = len(filter(lambda x: x.enter_sends, active_users))
|
||||
self.report_percentage(num_enter_sends, num_active,
|
||||
"active users have enter-sends")
|
||||
|
||||
@@ -123,29 +120,29 @@ class Command(BaseCommand):
|
||||
starrers = UserMessage.objects.filter(user_profile__in=user_profiles,
|
||||
flags=UserMessage.flags.starred).values(
|
||||
"user_profile").annotate(count=Count("user_profile"))
|
||||
print("%d users have starred %d messages" % (
|
||||
len(starrers), sum([elt["count"] for elt in starrers])))
|
||||
print "%d users have starred %d messages" % (
|
||||
len(starrers), sum([elt["count"] for elt in starrers]))
|
||||
|
||||
active_user_subs = Subscription.objects.filter(
|
||||
user_profile__in=user_profiles, active=True)
|
||||
|
||||
# Streams not in home view
|
||||
non_home_view = active_user_subs.filter(is_muted=True).values(
|
||||
non_home_view = active_user_subs.filter(in_home_view=False).values(
|
||||
"user_profile").annotate(count=Count("user_profile"))
|
||||
print("%d users have %d streams not in home view" % (
|
||||
len(non_home_view), sum([elt["count"] for elt in non_home_view])))
|
||||
print "%d users have %d streams not in home view" % (
|
||||
len(non_home_view), sum([elt["count"] for elt in non_home_view]))
|
||||
|
||||
# Code block markup
|
||||
markup_messages = human_messages.filter(
|
||||
sender__realm=realm, content__contains="~~~").values(
|
||||
"sender").annotate(count=Count("sender"))
|
||||
print("%d users have used code block markup on %s messages" % (
|
||||
len(markup_messages), sum([elt["count"] for elt in markup_messages])))
|
||||
print "%d users have used code block markup on %s messages" % (
|
||||
len(markup_messages), sum([elt["count"] for elt in markup_messages]))
|
||||
|
||||
# Notifications for stream messages
|
||||
notifications = active_user_subs.filter(desktop_notifications=True).values(
|
||||
notifications = active_user_subs.filter(notifications=True).values(
|
||||
"user_profile").annotate(count=Count("user_profile"))
|
||||
print("%d users receive desktop notifications for %d streams" % (
|
||||
len(notifications), sum([elt["count"] for elt in notifications])))
|
||||
print "%d users receive desktop notifications for %d streams" % (
|
||||
len(notifications), sum([elt["count"] for elt in notifications]))
|
||||
|
||||
print("")
|
||||
print ""
|
||||
|
||||
@@ -1,56 +1,36 @@
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
from __future__ import absolute_import
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models import Q
|
||||
|
||||
from zerver.models import Message, Realm, \
|
||||
Recipient, Stream, Subscription, get_realm
|
||||
from zerver.models import Realm, Stream, Message, Subscription, Recipient
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Generate statistics on the streams for a realm."
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
|
||||
help="realm to generate statistics for")
|
||||
|
||||
def handle(self, *args: Any, **options: str) -> None:
|
||||
if options['realms']:
|
||||
def handle(self, *args, **options):
|
||||
if args:
|
||||
try:
|
||||
realms = [get_realm(string_id) for string_id in options['realms']]
|
||||
except Realm.DoesNotExist as e:
|
||||
raise CommandError(e)
|
||||
realms = [Realm.objects.get(domain=domain) for domain in args]
|
||||
except Realm.DoesNotExist, e:
|
||||
print e
|
||||
exit(1)
|
||||
else:
|
||||
realms = Realm.objects.all()
|
||||
|
||||
for realm in realms:
|
||||
print realm.domain
|
||||
print "------------"
|
||||
print "%25s %15s %10s" % ("stream", "subscribers", "messages")
|
||||
streams = Stream.objects.filter(realm=realm).exclude(Q(name__istartswith="tutorial-"))
|
||||
# private stream count
|
||||
private_count = 0
|
||||
# public stream count
|
||||
public_count = 0
|
||||
invite_only_count = 0
|
||||
for stream in streams:
|
||||
if stream.invite_only:
|
||||
private_count += 1
|
||||
else:
|
||||
public_count += 1
|
||||
print("------------")
|
||||
print(realm.string_id, end=' ')
|
||||
print("%10s %d public streams and" % ("(", public_count), end=' ')
|
||||
print("%d private streams )" % (private_count,))
|
||||
print("------------")
|
||||
print("%25s %15s %10s %12s" % ("stream", "subscribers", "messages", "type"))
|
||||
|
||||
for stream in streams:
|
||||
if stream.invite_only:
|
||||
stream_type = 'private'
|
||||
else:
|
||||
stream_type = 'public'
|
||||
print("%25s" % (stream.name,), end=' ')
|
||||
invite_only_count += 1
|
||||
continue
|
||||
print "%25s" % (stream.name,),
|
||||
recipient = Recipient.objects.filter(type=Recipient.STREAM, type_id=stream.id)
|
||||
print("%10d" % (len(Subscription.objects.filter(recipient=recipient,
|
||||
active=True)),), end=' ')
|
||||
print "%10d" % (len(Subscription.objects.filter(recipient=recipient, active=True)),),
|
||||
num_messages = len(Message.objects.filter(recipient=recipient))
|
||||
print("%12d" % (num_messages,), end=' ')
|
||||
print("%15s" % (stream_type,))
|
||||
print("")
|
||||
print "%12d" % (num_messages,)
|
||||
print "%d invite-only streams" % (invite_only_count,)
|
||||
print ""
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any, Dict
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils.dateparse import parse_datetime
|
||||
from django.utils.timezone import now as timezone_now
|
||||
from django.utils.timezone import utc as timezone_utc
|
||||
|
||||
from analytics.lib.counts import COUNT_STATS, logger, process_count_stat
|
||||
from scripts.lib.zulip_tools import ENDC, WARNING
|
||||
from zerver.lib.remote_server import send_analytics_to_remote_server
|
||||
from zerver.lib.timestamp import floor_to_hour
|
||||
from zerver.models import Realm
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = """Fills Analytics tables.
|
||||
|
||||
Run as a cron job that runs every hour."""
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('--time', '-t',
|
||||
type=str,
|
||||
help='Update stat tables from current state to'
|
||||
'--time. Defaults to the current time.',
|
||||
default=timezone_now().isoformat())
|
||||
parser.add_argument('--utc',
|
||||
action='store_true',
|
||||
help="Interpret --time in UTC.",
|
||||
default=False)
|
||||
parser.add_argument('--stat', '-s',
|
||||
type=str,
|
||||
help="CountStat to process. If omitted, all stats are processed.")
|
||||
parser.add_argument('--verbose',
|
||||
action='store_true',
|
||||
help="Print timing information to stdout.",
|
||||
default=False)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
try:
|
||||
os.mkdir(settings.ANALYTICS_LOCK_DIR)
|
||||
except OSError:
|
||||
print(WARNING + "Analytics lock %s is unavailable; exiting... " + ENDC)
|
||||
return
|
||||
|
||||
try:
|
||||
self.run_update_analytics_counts(options)
|
||||
finally:
|
||||
os.rmdir(settings.ANALYTICS_LOCK_DIR)
|
||||
|
||||
def run_update_analytics_counts(self, options: Dict[str, Any]) -> None:
|
||||
# installation_epoch relies on there being at least one realm; we
|
||||
# shouldn't run the analytics code if that condition isn't satisfied
|
||||
if not Realm.objects.exists():
|
||||
logger.info("No realms, stopping update_analytics_counts")
|
||||
return
|
||||
|
||||
fill_to_time = parse_datetime(options['time'])
|
||||
if options['utc']:
|
||||
fill_to_time = fill_to_time.replace(tzinfo=timezone_utc)
|
||||
if fill_to_time.tzinfo is None:
|
||||
raise ValueError("--time must be timezone aware. Maybe you meant to use the --utc option?")
|
||||
|
||||
fill_to_time = floor_to_hour(fill_to_time.astimezone(timezone_utc))
|
||||
|
||||
if options['stat'] is not None:
|
||||
stats = [COUNT_STATS[options['stat']]]
|
||||
else:
|
||||
stats = list(COUNT_STATS.values())
|
||||
|
||||
logger.info("Starting updating analytics counts through %s" % (fill_to_time,))
|
||||
if options['verbose']:
|
||||
start = time.time()
|
||||
last = start
|
||||
|
||||
for stat in stats:
|
||||
process_count_stat(stat, fill_to_time)
|
||||
if options['verbose']:
|
||||
print("Updated %s in %.3fs" % (stat.property, time.time() - last))
|
||||
last = time.time()
|
||||
|
||||
if options['verbose']:
|
||||
print("Finished updating analytics counts through %s in %.3fs" %
|
||||
(fill_to_time, time.time() - start))
|
||||
logger.info("Finished updating analytics counts through %s" % (fill_to_time,))
|
||||
|
||||
if settings.PUSH_NOTIFICATION_BOUNCER_URL and settings.SUBMIT_USAGE_STATISTICS:
|
||||
send_analytics_to_remote_server()
|
||||
@@ -1,41 +1,37 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import datetime
|
||||
from argparse import ArgumentParser
|
||||
from typing import Any
|
||||
import pytz
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.utils.timezone import now as timezone_now
|
||||
|
||||
from zerver.models import Message, Realm, Stream, UserProfile, get_realm
|
||||
from django.core.management.base import BaseCommand
|
||||
from zerver.models import UserProfile, Realm, Stream, Message
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Generate statistics on user activity."
|
||||
|
||||
def add_arguments(self, parser: ArgumentParser) -> None:
|
||||
parser.add_argument('realms', metavar='<realm>', type=str, nargs='*',
|
||||
help="realm to generate statistics for")
|
||||
def messages_sent_by(self, user, week):
|
||||
start = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=(week + 1)*7)
|
||||
end = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=week*7)
|
||||
return Message.objects.filter(sender=user, pub_date__gt=start, pub_date__lte=end).count()
|
||||
|
||||
def messages_sent_by(self, user: UserProfile, week: int) -> int:
|
||||
start = timezone_now() - datetime.timedelta(days=(week + 1)*7)
|
||||
end = timezone_now() - datetime.timedelta(days=week*7)
|
||||
return Message.objects.filter(sender=user, date_sent__gt=start, date_sent__lte=end).count()
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
if options['realms']:
|
||||
def handle(self, *args, **options):
|
||||
if args:
|
||||
try:
|
||||
realms = [get_realm(string_id) for string_id in options['realms']]
|
||||
except Realm.DoesNotExist as e:
|
||||
raise CommandError(e)
|
||||
realms = [Realm.objects.get(domain=domain) for domain in args]
|
||||
except Realm.DoesNotExist, e:
|
||||
print e
|
||||
exit(1)
|
||||
else:
|
||||
realms = Realm.objects.all()
|
||||
|
||||
for realm in realms:
|
||||
print(realm.string_id)
|
||||
print realm.domain
|
||||
user_profiles = UserProfile.objects.filter(realm=realm, is_active=True)
|
||||
print("%d users" % (len(user_profiles),))
|
||||
print("%d streams" % (len(Stream.objects.filter(realm=realm)),))
|
||||
print "%d users" % (len(user_profiles),)
|
||||
print "%d streams" % (len(Stream.objects.filter(realm=realm)),)
|
||||
|
||||
for user_profile in user_profiles:
|
||||
print("%35s" % (user_profile.email,), end=' ')
|
||||
print "%35s" % (user_profile.email,),
|
||||
for week in range(10):
|
||||
print("%5d" % (self.messages_sent_by(user_profile, week),), end=' ')
|
||||
print("")
|
||||
print "%5d" % (self.messages_sent_by(user_profile, week)),
|
||||
print ""
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('zerver', '0030_realm_org_type'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Anomaly',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('info', models.CharField(max_length=1000)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='HuddleCount',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('huddle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Recipient')),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
('property', models.CharField(max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('interval', models.CharField(max_length=20)),
|
||||
('value', models.BigIntegerField()),
|
||||
('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='InstallationCount',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('property', models.CharField(max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('interval', models.CharField(max_length=20)),
|
||||
('value', models.BigIntegerField()),
|
||||
('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='RealmCount',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
|
||||
('property', models.CharField(max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('interval', models.CharField(max_length=20)),
|
||||
('value', models.BigIntegerField()),
|
||||
('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
|
||||
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='StreamCount',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
|
||||
('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Stream')),
|
||||
('property', models.CharField(max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('interval', models.CharField(max_length=20)),
|
||||
('value', models.BigIntegerField()),
|
||||
('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='UserCount',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('realm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zerver.Realm')),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
('property', models.CharField(max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('interval', models.CharField(max_length=20)),
|
||||
('value', models.BigIntegerField()),
|
||||
('anomaly', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analytics.Anomaly', null=True)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='usercount',
|
||||
unique_together=set([('user', 'property', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='streamcount',
|
||||
unique_together=set([('stream', 'property', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='realmcount',
|
||||
unique_together=set([('realm', 'property', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='installationcount',
|
||||
unique_together=set([('property', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='huddlecount',
|
||||
unique_together=set([('huddle', 'property', 'end_time', 'interval')]),
|
||||
),
|
||||
]
|
||||
@@ -1,30 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterUniqueTogether(
|
||||
name='huddlecount',
|
||||
unique_together=set([]),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='huddlecount',
|
||||
name='anomaly',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='huddlecount',
|
||||
name='huddle',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='huddlecount',
|
||||
name='user',
|
||||
),
|
||||
migrations.DeleteModel(
|
||||
name='HuddleCount',
|
||||
),
|
||||
]
|
||||
@@ -1,22 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0002_remove_huddlecount'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='FillState',
|
||||
fields=[
|
||||
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
|
||||
('property', models.CharField(unique=True, max_length=40)),
|
||||
('end_time', models.DateTimeField()),
|
||||
('state', models.PositiveSmallIntegerField()),
|
||||
('last_modified', models.DateTimeField(auto_now=True)),
|
||||
],
|
||||
bases=(models.Model,),
|
||||
),
|
||||
]
|
||||
@@ -1,31 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0003_fillstate'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='installationcount',
|
||||
name='subgroup',
|
||||
field=models.CharField(max_length=16, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='realmcount',
|
||||
name='subgroup',
|
||||
field=models.CharField(max_length=16, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='streamcount',
|
||||
name='subgroup',
|
||||
field=models.CharField(max_length=16, null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='usercount',
|
||||
name='subgroup',
|
||||
field=models.CharField(max_length=16, null=True),
|
||||
),
|
||||
]
|
||||
@@ -1,51 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations, models
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0004_add_subgroup'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='installationcount',
|
||||
name='interval',
|
||||
field=models.CharField(max_length=8),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='installationcount',
|
||||
name='property',
|
||||
field=models.CharField(max_length=32),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='realmcount',
|
||||
name='interval',
|
||||
field=models.CharField(max_length=8),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='realmcount',
|
||||
name='property',
|
||||
field=models.CharField(max_length=32),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='streamcount',
|
||||
name='interval',
|
||||
field=models.CharField(max_length=8),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='streamcount',
|
||||
name='property',
|
||||
field=models.CharField(max_length=32),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='usercount',
|
||||
name='interval',
|
||||
field=models.CharField(max_length=8),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='usercount',
|
||||
name='property',
|
||||
field=models.CharField(max_length=32),
|
||||
),
|
||||
]
|
||||
@@ -1,27 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0005_alter_field_size'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterUniqueTogether(
|
||||
name='installationcount',
|
||||
unique_together=set([('property', 'subgroup', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='realmcount',
|
||||
unique_together=set([('realm', 'property', 'subgroup', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='streamcount',
|
||||
unique_together=set([('stream', 'property', 'subgroup', 'end_time', 'interval')]),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='usercount',
|
||||
unique_together=set([('user', 'property', 'subgroup', 'end_time', 'interval')]),
|
||||
),
|
||||
]
|
||||
@@ -1,44 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.10.4 on 2017-01-16 20:50
|
||||
from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0006_add_subgroup_to_unique_constraints'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterUniqueTogether(
|
||||
name='installationcount',
|
||||
unique_together=set([('property', 'subgroup', 'end_time')]),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='installationcount',
|
||||
name='interval',
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='realmcount',
|
||||
unique_together=set([('realm', 'property', 'subgroup', 'end_time')]),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='realmcount',
|
||||
name='interval',
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='streamcount',
|
||||
unique_together=set([('stream', 'property', 'subgroup', 'end_time')]),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='streamcount',
|
||||
name='interval',
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name='usercount',
|
||||
unique_together=set([('user', 'property', 'subgroup', 'end_time')]),
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='usercount',
|
||||
name='interval',
|
||||
),
|
||||
]
|
||||
@@ -1,25 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.10.5 on 2017-02-01 22:28
|
||||
from django.db import migrations
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('zerver', '0050_userprofile_avatar_version'),
|
||||
('analytics', '0007_remove_interval'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterIndexTogether(
|
||||
name='realmcount',
|
||||
index_together=set([('property', 'end_time')]),
|
||||
),
|
||||
migrations.AlterIndexTogether(
|
||||
name='streamcount',
|
||||
index_together=set([('property', 'realm', 'end_time')]),
|
||||
),
|
||||
migrations.AlterIndexTogether(
|
||||
name='usercount',
|
||||
index_together=set([('property', 'realm', 'end_time')]),
|
||||
),
|
||||
]
|
||||
@@ -1,28 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations
|
||||
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
|
||||
from django.db.migrations.state import StateApps
|
||||
|
||||
def delete_messages_sent_to_stream_stat(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
|
||||
UserCount = apps.get_model('analytics', 'UserCount')
|
||||
StreamCount = apps.get_model('analytics', 'StreamCount')
|
||||
RealmCount = apps.get_model('analytics', 'RealmCount')
|
||||
InstallationCount = apps.get_model('analytics', 'InstallationCount')
|
||||
FillState = apps.get_model('analytics', 'FillState')
|
||||
|
||||
property = 'messages_sent_to_stream:is_bot'
|
||||
UserCount.objects.filter(property=property).delete()
|
||||
StreamCount.objects.filter(property=property).delete()
|
||||
RealmCount.objects.filter(property=property).delete()
|
||||
InstallationCount.objects.filter(property=property).delete()
|
||||
FillState.objects.filter(property=property).delete()
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0008_add_count_indexes'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(delete_messages_sent_to_stream_stat),
|
||||
]
|
||||
@@ -1,26 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations
|
||||
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
|
||||
from django.db.migrations.state import StateApps
|
||||
|
||||
def clear_message_sent_by_message_type_values(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
|
||||
UserCount = apps.get_model('analytics', 'UserCount')
|
||||
StreamCount = apps.get_model('analytics', 'StreamCount')
|
||||
RealmCount = apps.get_model('analytics', 'RealmCount')
|
||||
InstallationCount = apps.get_model('analytics', 'InstallationCount')
|
||||
FillState = apps.get_model('analytics', 'FillState')
|
||||
|
||||
property = 'messages_sent:message_type:day'
|
||||
UserCount.objects.filter(property=property).delete()
|
||||
StreamCount.objects.filter(property=property).delete()
|
||||
RealmCount.objects.filter(property=property).delete()
|
||||
InstallationCount.objects.filter(property=property).delete()
|
||||
FillState.objects.filter(property=property).delete()
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [('analytics', '0009_remove_messages_to_stream_stat')]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(clear_message_sent_by_message_type_values),
|
||||
]
|
||||
@@ -1,27 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from django.db import migrations
|
||||
from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
|
||||
from django.db.migrations.state import StateApps
|
||||
|
||||
def clear_analytics_tables(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
|
||||
UserCount = apps.get_model('analytics', 'UserCount')
|
||||
StreamCount = apps.get_model('analytics', 'StreamCount')
|
||||
RealmCount = apps.get_model('analytics', 'RealmCount')
|
||||
InstallationCount = apps.get_model('analytics', 'InstallationCount')
|
||||
FillState = apps.get_model('analytics', 'FillState')
|
||||
|
||||
UserCount.objects.all().delete()
|
||||
StreamCount.objects.all().delete()
|
||||
RealmCount.objects.all().delete()
|
||||
InstallationCount.objects.all().delete()
|
||||
FillState.objects.all().delete()
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0010_clear_messages_sent_values'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(clear_analytics_tables),
|
||||
]
|
||||
@@ -1,36 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.6 on 2018-01-29 08:14
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0011_clear_analytics_tables'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='installationcount',
|
||||
name='anomaly',
|
||||
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='realmcount',
|
||||
name='anomaly',
|
||||
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='streamcount',
|
||||
name='anomaly',
|
||||
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='usercount',
|
||||
name='anomaly',
|
||||
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='analytics.Anomaly'),
|
||||
),
|
||||
]
|
||||
@@ -1,34 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.18 on 2019-02-02 02:47
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('analytics', '0012_add_on_delete'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='installationcount',
|
||||
name='anomaly',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='realmcount',
|
||||
name='anomaly',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='streamcount',
|
||||
name='anomaly',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='usercount',
|
||||
name='anomaly',
|
||||
),
|
||||
migrations.DeleteModel(
|
||||
name='Anomaly',
|
||||
),
|
||||
]
|
||||
@@ -1,92 +0,0 @@
|
||||
import datetime
|
||||
from typing import Optional
|
||||
|
||||
from django.db import models
|
||||
|
||||
from zerver.lib.timestamp import floor_to_day
|
||||
from zerver.models import Realm, Stream, UserProfile
|
||||
|
||||
class FillState(models.Model):
|
||||
property = models.CharField(max_length=40, unique=True) # type: str
|
||||
end_time = models.DateTimeField() # type: datetime.datetime
|
||||
|
||||
# Valid states are {DONE, STARTED}
|
||||
DONE = 1
|
||||
STARTED = 2
|
||||
state = models.PositiveSmallIntegerField() # type: int
|
||||
|
||||
last_modified = models.DateTimeField(auto_now=True) # type: datetime.datetime
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<FillState: %s %s %s>" % (self.property, self.end_time, self.state)
|
||||
|
||||
# The earliest/starting end_time in FillState
|
||||
# We assume there is at least one realm
|
||||
def installation_epoch() -> datetime.datetime:
|
||||
earliest_realm_creation = Realm.objects.aggregate(models.Min('date_created'))['date_created__min']
|
||||
return floor_to_day(earliest_realm_creation)
|
||||
|
||||
def last_successful_fill(property: str) -> Optional[datetime.datetime]:
|
||||
fillstate = FillState.objects.filter(property=property).first()
|
||||
if fillstate is None:
|
||||
return None
|
||||
if fillstate.state == FillState.DONE:
|
||||
return fillstate.end_time
|
||||
return fillstate.end_time - datetime.timedelta(hours=1)
|
||||
|
||||
class BaseCount(models.Model):
|
||||
# Note: When inheriting from BaseCount, you may want to rearrange
|
||||
# the order of the columns in the migration to make sure they
|
||||
# match how you'd like the table to be arranged.
|
||||
property = models.CharField(max_length=32) # type: str
|
||||
subgroup = models.CharField(max_length=16, null=True) # type: Optional[str]
|
||||
end_time = models.DateTimeField() # type: datetime.datetime
|
||||
value = models.BigIntegerField() # type: int
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
class InstallationCount(BaseCount):
|
||||
|
||||
class Meta:
|
||||
unique_together = ("property", "subgroup", "end_time")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<InstallationCount: %s %s %s>" % (self.property, self.subgroup, self.value)
|
||||
|
||||
class RealmCount(BaseCount):
|
||||
realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
|
||||
|
||||
class Meta:
|
||||
unique_together = ("realm", "property", "subgroup", "end_time")
|
||||
index_together = ["property", "end_time"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<RealmCount: %s %s %s %s>" % (self.realm, self.property, self.subgroup, self.value)
|
||||
|
||||
class UserCount(BaseCount):
|
||||
user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
|
||||
realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
|
||||
|
||||
class Meta:
|
||||
unique_together = ("user", "property", "subgroup", "end_time")
|
||||
# This index dramatically improves the performance of
|
||||
# aggregating from users to realms
|
||||
index_together = ["property", "realm", "end_time"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<UserCount: %s %s %s %s>" % (self.user, self.property, self.subgroup, self.value)
|
||||
|
||||
class StreamCount(BaseCount):
|
||||
stream = models.ForeignKey(Stream, on_delete=models.CASCADE)
|
||||
realm = models.ForeignKey(Realm, on_delete=models.CASCADE)
|
||||
|
||||
class Meta:
|
||||
unique_together = ("stream", "property", "subgroup", "end_time")
|
||||
# This index dramatically improves the performance of
|
||||
# aggregating from streams to realms
|
||||
index_together = ["property", "realm", "end_time"]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return "<StreamCount: %s %s %s %s %s>" % (
|
||||
self.stream, self.property, self.subgroup, self.value, self.id)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,29 +0,0 @@
|
||||
from analytics.lib.counts import CountStat
|
||||
from analytics.lib.fixtures import generate_time_series_data
|
||||
from zerver.lib.test_classes import ZulipTestCase
|
||||
|
||||
# A very light test suite; the code being tested is not run in production.
|
||||
class TestFixtures(ZulipTestCase):
|
||||
def test_deterministic_settings(self) -> None:
|
||||
# test basic business_hour / non_business_hour calculation
|
||||
# test we get an array of the right length with frequency=CountStat.DAY
|
||||
data = generate_time_series_data(
|
||||
days=7, business_hours_base=20, non_business_hours_base=15, spikiness=0)
|
||||
self.assertEqual(data, [400, 400, 400, 400, 400, 360, 360])
|
||||
|
||||
data = generate_time_series_data(
|
||||
days=1, business_hours_base=2000, non_business_hours_base=1500,
|
||||
growth=2, spikiness=0, frequency=CountStat.HOUR)
|
||||
# test we get an array of the right length with frequency=CountStat.HOUR
|
||||
self.assertEqual(len(data), 24)
|
||||
# test that growth doesn't affect the first data point
|
||||
self.assertEqual(data[0], 2000)
|
||||
# test that the last data point is growth times what it otherwise would be
|
||||
self.assertEqual(data[-1], 1500*2)
|
||||
|
||||
# test autocorrelation == 1, since that's the easiest value to test
|
||||
data = generate_time_series_data(
|
||||
days=1, business_hours_base=2000, non_business_hours_base=2000,
|
||||
autocorrelation=1, frequency=CountStat.HOUR)
|
||||
self.assertEqual(data[0], data[1])
|
||||
self.assertEqual(data[0], data[-1])
|
||||
@@ -1,618 +0,0 @@
|
||||
from datetime import datetime, timedelta
|
||||
from typing import List, Optional
|
||||
|
||||
import mock
|
||||
from django.utils.timezone import utc
|
||||
from django.http import HttpResponse
|
||||
import ujson
|
||||
|
||||
from analytics.lib.counts import COUNT_STATS, CountStat
|
||||
from analytics.lib.time_utils import time_range
|
||||
from analytics.models import FillState, \
|
||||
RealmCount, UserCount, last_successful_fill
|
||||
from analytics.views import rewrite_client_arrays, \
|
||||
sort_by_totals, sort_client_labels
|
||||
from zerver.lib.test_classes import ZulipTestCase
|
||||
from zerver.lib.timestamp import ceiling_to_day, \
|
||||
ceiling_to_hour, datetime_to_timestamp
|
||||
from zerver.lib.actions import do_create_multiuse_invite_link, \
|
||||
do_send_realm_reactivation_email
|
||||
from zerver.models import Client, get_realm, MultiuseInvite
|
||||
|
||||
class TestStatsEndpoint(ZulipTestCase):
|
||||
def test_stats(self) -> None:
|
||||
self.user = self.example_user('hamlet')
|
||||
self.login(self.user.email)
|
||||
result = self.client_get('/stats')
|
||||
self.assertEqual(result.status_code, 200)
|
||||
# Check that we get something back
|
||||
self.assert_in_response("Zulip analytics for", result)
|
||||
|
||||
def test_guest_user_cant_access_stats(self) -> None:
|
||||
self.user = self.example_user('polonius')
|
||||
self.login(self.user.email)
|
||||
result = self.client_get('/stats')
|
||||
self.assert_json_error(result, "Not allowed for guest users", 400)
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data')
|
||||
self.assert_json_error(result, "Not allowed for guest users", 400)
|
||||
|
||||
def test_stats_for_realm(self) -> None:
|
||||
user_profile = self.example_user('hamlet')
|
||||
self.login(user_profile.email)
|
||||
|
||||
result = self.client_get('/stats/realm/zulip/')
|
||||
self.assertEqual(result.status_code, 302)
|
||||
|
||||
user_profile = self.example_user('hamlet')
|
||||
user_profile.is_staff = True
|
||||
user_profile.save(update_fields=['is_staff'])
|
||||
|
||||
result = self.client_get('/stats/realm/not_existing_realm/')
|
||||
self.assertEqual(result.status_code, 302)
|
||||
|
||||
result = self.client_get('/stats/realm/zulip/')
|
||||
self.assertEqual(result.status_code, 200)
|
||||
self.assert_in_response("Zulip analytics for", result)
|
||||
|
||||
def test_stats_for_installation(self) -> None:
|
||||
user_profile = self.example_user('hamlet')
|
||||
self.login(user_profile.email)
|
||||
|
||||
result = self.client_get('/stats/installation')
|
||||
self.assertEqual(result.status_code, 302)
|
||||
|
||||
user_profile = self.example_user('hamlet')
|
||||
user_profile.is_staff = True
|
||||
user_profile.save(update_fields=['is_staff'])
|
||||
|
||||
result = self.client_get('/stats/installation')
|
||||
self.assertEqual(result.status_code, 200)
|
||||
self.assert_in_response("Zulip analytics for", result)
|
||||
|
||||
class TestGetChartData(ZulipTestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.realm = get_realm('zulip')
|
||||
self.user = self.example_user('hamlet')
|
||||
self.login(self.user.email)
|
||||
self.end_times_hour = [ceiling_to_hour(self.realm.date_created) + timedelta(hours=i)
|
||||
for i in range(4)]
|
||||
self.end_times_day = [ceiling_to_day(self.realm.date_created) + timedelta(days=i)
|
||||
for i in range(4)]
|
||||
|
||||
def data(self, i: int) -> List[int]:
|
||||
return [0, 0, i, 0]
|
||||
|
||||
def insert_data(self, stat: CountStat, realm_subgroups: List[Optional[str]],
|
||||
user_subgroups: List[str]) -> None:
|
||||
if stat.frequency == CountStat.HOUR:
|
||||
insert_time = self.end_times_hour[2]
|
||||
fill_time = self.end_times_hour[-1]
|
||||
if stat.frequency == CountStat.DAY:
|
||||
insert_time = self.end_times_day[2]
|
||||
fill_time = self.end_times_day[-1]
|
||||
|
||||
RealmCount.objects.bulk_create([
|
||||
RealmCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
|
||||
value=100+i, realm=self.realm)
|
||||
for i, subgroup in enumerate(realm_subgroups)])
|
||||
UserCount.objects.bulk_create([
|
||||
UserCount(property=stat.property, subgroup=subgroup, end_time=insert_time,
|
||||
value=200+i, realm=self.realm, user=self.user)
|
||||
for i, subgroup in enumerate(user_subgroups)])
|
||||
FillState.objects.create(property=stat.property, end_time=fill_time, state=FillState.DONE)
|
||||
|
||||
def test_number_of_humans(self) -> None:
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['1day_actives::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['active_users_audit:is_bot:day']
|
||||
self.insert_data(stat, ['false'], [])
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data, {
|
||||
'msg': '',
|
||||
'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
|
||||
'frequency': CountStat.DAY,
|
||||
'everyone': {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)},
|
||||
'display_order': None,
|
||||
'result': 'success',
|
||||
})
|
||||
|
||||
def test_messages_sent_over_time(self) -> None:
|
||||
stat = COUNT_STATS['messages_sent:is_bot:hour']
|
||||
self.insert_data(stat, ['true', 'false'], ['false'])
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_over_time'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data, {
|
||||
'msg': '',
|
||||
'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_hour],
|
||||
'frequency': CountStat.HOUR,
|
||||
'everyone': {'bot': self.data(100), 'human': self.data(101)},
|
||||
'user': {'bot': self.data(0), 'human': self.data(200)},
|
||||
'display_order': None,
|
||||
'result': 'success',
|
||||
})
|
||||
|
||||
def test_messages_sent_by_message_type(self) -> None:
|
||||
stat = COUNT_STATS['messages_sent:message_type:day']
|
||||
self.insert_data(stat, ['public_stream', 'private_message'],
|
||||
['public_stream', 'private_stream'])
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_by_message_type'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data, {
|
||||
'msg': '',
|
||||
'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
|
||||
'frequency': CountStat.DAY,
|
||||
'everyone': {'Public streams': self.data(100), 'Private streams': self.data(0),
|
||||
'Private messages': self.data(101), 'Group private messages': self.data(0)},
|
||||
'user': {'Public streams': self.data(200), 'Private streams': self.data(201),
|
||||
'Private messages': self.data(0), 'Group private messages': self.data(0)},
|
||||
'display_order': ['Private messages', 'Public streams', 'Private streams', 'Group private messages'],
|
||||
'result': 'success',
|
||||
})
|
||||
|
||||
def test_messages_sent_by_client(self) -> None:
|
||||
stat = COUNT_STATS['messages_sent:client:day']
|
||||
client1 = Client.objects.create(name='client 1')
|
||||
client2 = Client.objects.create(name='client 2')
|
||||
client3 = Client.objects.create(name='client 3')
|
||||
client4 = Client.objects.create(name='client 4')
|
||||
self.insert_data(stat, [client4.id, client3.id, client2.id],
|
||||
[client3.id, client1.id])
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_by_client'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data, {
|
||||
'msg': '',
|
||||
'end_times': [datetime_to_timestamp(dt) for dt in self.end_times_day],
|
||||
'frequency': CountStat.DAY,
|
||||
'everyone': {'client 4': self.data(100), 'client 3': self.data(101),
|
||||
'client 2': self.data(102)},
|
||||
'user': {'client 3': self.data(200), 'client 1': self.data(201)},
|
||||
'display_order': ['client 1', 'client 2', 'client 3', 'client 4'],
|
||||
'result': 'success',
|
||||
})
|
||||
|
||||
def test_include_empty_subgroups(self) -> None:
|
||||
FillState.objects.create(
|
||||
property='realm_active_humans::day', end_time=self.end_times_day[0],
|
||||
state=FillState.DONE)
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['everyone'], {"_1day": [0], "_15day": [0], "all_time": [0]})
|
||||
self.assertFalse('user' in data)
|
||||
|
||||
FillState.objects.create(
|
||||
property='messages_sent:is_bot:hour', end_time=self.end_times_hour[0],
|
||||
state=FillState.DONE)
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_over_time'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['everyone'], {'human': [0], 'bot': [0]})
|
||||
self.assertEqual(data['user'], {'human': [0], 'bot': [0]})
|
||||
|
||||
FillState.objects.create(
|
||||
property='messages_sent:message_type:day', end_time=self.end_times_day[0],
|
||||
state=FillState.DONE)
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_by_message_type'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['everyone'], {
|
||||
'Public streams': [0], 'Private streams': [0],
|
||||
'Private messages': [0], 'Group private messages': [0]})
|
||||
self.assertEqual(data['user'], {
|
||||
'Public streams': [0], 'Private streams': [0],
|
||||
'Private messages': [0], 'Group private messages': [0]})
|
||||
|
||||
FillState.objects.create(
|
||||
property='messages_sent:client:day', end_time=self.end_times_day[0],
|
||||
state=FillState.DONE)
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'messages_sent_by_client'})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['everyone'], {})
|
||||
self.assertEqual(data['user'], {})
|
||||
|
||||
def test_start_and_end(self) -> None:
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['1day_actives::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['active_users_audit:is_bot:day']
|
||||
self.insert_data(stat, ['false'], [])
|
||||
end_time_timestamps = [datetime_to_timestamp(dt) for dt in self.end_times_day]
|
||||
|
||||
# valid start and end
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans',
|
||||
'start': end_time_timestamps[1],
|
||||
'end': end_time_timestamps[2]})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['end_times'], end_time_timestamps[1:3])
|
||||
self.assertEqual(data['everyone'], {'_1day': [0, 100], '_15day': [0, 100], 'all_time': [0, 100]})
|
||||
|
||||
# start later then end
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans',
|
||||
'start': end_time_timestamps[2],
|
||||
'end': end_time_timestamps[1]})
|
||||
self.assert_json_error_contains(result, 'Start time is later than')
|
||||
|
||||
def test_min_length(self) -> None:
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['1day_actives::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
stat = COUNT_STATS['active_users_audit:is_bot:day']
|
||||
self.insert_data(stat, ['false'], [])
|
||||
# test min_length is too short to change anything
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans',
|
||||
'min_length': 2})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in self.end_times_day])
|
||||
self.assertEqual(data['everyone'], {'_1day': self.data(100), '_15day': self.data(100), 'all_time': self.data(100)})
|
||||
# test min_length larger than filled data
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans',
|
||||
'min_length': 5})
|
||||
self.assert_json_success(result)
|
||||
data = result.json()
|
||||
end_times = [ceiling_to_day(self.realm.date_created) + timedelta(days=i) for i in range(-1, 4)]
|
||||
self.assertEqual(data['end_times'], [datetime_to_timestamp(dt) for dt in end_times])
|
||||
self.assertEqual(data['everyone'], {'_1day': [0]+self.data(100), '_15day': [0]+self.data(100), 'all_time': [0]+self.data(100)})
|
||||
|
||||
def test_non_existent_chart(self) -> None:
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'does_not_exist'})
|
||||
self.assert_json_error_contains(result, 'Unknown chart name')
|
||||
|
||||
def test_analytics_not_running(self) -> None:
|
||||
# try to get data for a valid chart, but before we've put anything in the database
|
||||
# (e.g. before update_analytics_counts has been run)
|
||||
with mock.patch('logging.warning'):
|
||||
result = self.client_get('/json/analytics/chart_data',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_error_contains(result, 'No analytics data available')
|
||||
|
||||
def test_get_chart_data_for_realm(self) -> None:
|
||||
user_profile = self.example_user('hamlet')
|
||||
self.login(user_profile.email)
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data/realm/zulip/',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_error(result, "Must be an server administrator", 400)
|
||||
|
||||
user_profile = self.example_user('hamlet')
|
||||
user_profile.is_staff = True
|
||||
user_profile.save(update_fields=['is_staff'])
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data/realm/not_existing_realm',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_error(result, 'Invalid organization', 400)
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data/realm/zulip',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_success(result)
|
||||
|
||||
def test_get_chart_data_for_installation(self) -> None:
|
||||
user_profile = self.example_user('hamlet')
|
||||
self.login(user_profile.email)
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data/installation',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_error(result, "Must be an server administrator", 400)
|
||||
|
||||
user_profile = self.example_user('hamlet')
|
||||
user_profile.is_staff = True
|
||||
user_profile.save(update_fields=['is_staff'])
|
||||
stat = COUNT_STATS['realm_active_humans::day']
|
||||
self.insert_data(stat, [None], [])
|
||||
|
||||
result = self.client_get('/json/analytics/chart_data/installation',
|
||||
{'chart_name': 'number_of_humans'})
|
||||
self.assert_json_success(result)
|
||||
|
||||
class TestSupportEndpoint(ZulipTestCase):
|
||||
def test_search(self) -> None:
|
||||
def check_hamlet_user_query_result(result: HttpResponse) -> None:
|
||||
self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>',
|
||||
'<b>Email</b>: hamlet@zulip.com', '<b>Is active</b>: True<br>',
|
||||
'<b>Admins</b>: iago@zulip.com\n',
|
||||
'class="copy-button" data-copytext="iago@zulip.com"'
|
||||
], result)
|
||||
|
||||
def check_zulip_realm_query_result(result: HttpResponse) -> None:
|
||||
zulip_realm = get_realm("zulip")
|
||||
self.assert_in_success_response(['<input type="hidden" name="realm_id" value="%s"' % (zulip_realm.id,),
|
||||
'Zulip Dev</h3>',
|
||||
'<option value="1" selected>Self Hosted</option>',
|
||||
'<option value="2" >Limited</option>',
|
||||
'input type="number" name="discount" value="None"',
|
||||
'<option value="active" selected>Active</option>',
|
||||
'<option value="deactivated" >Deactivated</option>',
|
||||
'scrub-realm-button">',
|
||||
'data-string-id="zulip"'], result)
|
||||
|
||||
def check_lear_realm_query_result(result: HttpResponse) -> None:
|
||||
lear_realm = get_realm("lear")
|
||||
self.assert_in_success_response(['<input type="hidden" name="realm_id" value="%s"' % (lear_realm.id,),
|
||||
'Lear & Co.</h3>',
|
||||
'<option value="1" selected>Self Hosted</option>',
|
||||
'<option value="2" >Limited</option>',
|
||||
'input type="number" name="discount" value="None"',
|
||||
'<option value="active" selected>Active</option>',
|
||||
'<option value="deactivated" >Deactivated</option>',
|
||||
'scrub-realm-button">',
|
||||
'data-string-id="lear"'], result)
|
||||
|
||||
def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: Optional[bool]=False) -> None:
|
||||
self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
|
||||
'<b>Email</b>: {}'.format(email),
|
||||
], result)
|
||||
if invite:
|
||||
self.assert_in_success_response(['<span class="label">invite</span>'], result)
|
||||
self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3',
|
||||
'<b>Status</b>: Link has never been clicked'], result)
|
||||
self.assert_in_success_response([], result)
|
||||
else:
|
||||
self.assert_not_in_success_response(['<span class="label">invite</span>'], result)
|
||||
self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day',
|
||||
'<b>Status</b>: Link has never been clicked'], result)
|
||||
|
||||
def check_realm_creation_query_result(result: HttpResponse, email: str) -> None:
|
||||
self.assert_in_success_response(['<span class="label">preregistration user</span>\n',
|
||||
'<span class="label">realm creation</span>\n',
|
||||
'<b>Link</b>: http://zulip.testserver/accounts/do_confirm/',
|
||||
'<b>Expires in</b>: 1\xa0day<br>\n'
|
||||
], result)
|
||||
|
||||
def check_multiuse_invite_link_query_result(result: HttpResponse) -> None:
|
||||
self.assert_in_success_response(['<span class="label">multiuse invite</span>\n',
|
||||
'<b>Link</b>: http://zulip.testserver/join/',
|
||||
'<b>Expires in</b>: 1\xa0week, 3'
|
||||
], result)
|
||||
|
||||
def check_realm_reactivation_link_query_result(result: HttpResponse) -> None:
|
||||
self.assert_in_success_response(['<span class="label">realm reactivation</span>\n',
|
||||
'<b>Link</b>: http://zulip.testserver/reactivate/',
|
||||
'<b>Expires in</b>: 1\xa0day'
|
||||
], result)
|
||||
|
||||
cordelia_email = self.example_email("cordelia")
|
||||
self.login(cordelia_email)
|
||||
|
||||
result = self.client_get("/activity/support")
|
||||
self.assertEqual(result.status_code, 302)
|
||||
self.assertEqual(result["Location"], "/login/")
|
||||
|
||||
iago_email = self.example_email("iago")
|
||||
self.login(iago_email)
|
||||
|
||||
result = self.client_get("/activity/support")
|
||||
self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result)
|
||||
|
||||
result = self.client_get("/activity/support", {"q": "hamlet@zulip.com"})
|
||||
check_hamlet_user_query_result(result)
|
||||
check_zulip_realm_query_result(result)
|
||||
|
||||
result = self.client_get("/activity/support", {"q": "lear"})
|
||||
check_lear_realm_query_result(result)
|
||||
|
||||
result = self.client_get("/activity/support", {"q": "http://lear.testserver"})
|
||||
check_lear_realm_query_result(result)
|
||||
|
||||
with self.settings(REALM_HOSTS={'zulip': 'localhost'}):
|
||||
result = self.client_get("/activity/support", {"q": "http://localhost"})
|
||||
check_zulip_realm_query_result(result)
|
||||
|
||||
result = self.client_get("/activity/support", {"q": "hamlet@zulip.com, lear"})
|
||||
check_hamlet_user_query_result(result)
|
||||
check_zulip_realm_query_result(result)
|
||||
check_lear_realm_query_result(result)
|
||||
|
||||
result = self.client_get("/activity/support", {"q": "lear, Hamlet <hamlet@zulip.com>"})
|
||||
check_hamlet_user_query_result(result)
|
||||
check_zulip_realm_query_result(result)
|
||||
check_lear_realm_query_result(result)
|
||||
|
||||
self.client_post('/accounts/home/', {'email': self.nonreg_email("test")})
|
||||
self.login(iago_email)
|
||||
result = self.client_get("/activity/support", {"q": self.nonreg_email("test")})
|
||||
check_preregistration_user_query_result(result, self.nonreg_email("test"))
|
||||
check_zulip_realm_query_result(result)
|
||||
|
||||
stream_ids = [self.get_stream_id("Denmark")]
|
||||
invitee_emails = [self.nonreg_email("test1")]
|
||||
self.client_post("/json/invites", {"invitee_emails": invitee_emails,
|
||||
"stream_ids": ujson.dumps(stream_ids), "invite_as": 1})
|
||||
result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")})
|
||||
check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True)
|
||||
check_zulip_realm_query_result(result)
|
||||
|
||||
email = self.nonreg_email('alice')
|
||||
self.client_post('/new/', {'email': email})
|
||||
result = self.client_get("/activity/support", {"q": email})
|
||||
check_realm_creation_query_result(result, email)
|
||||
|
||||
do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1)
|
||||
result = self.client_get("/activity/support", {"q": "zulip"})
|
||||
check_multiuse_invite_link_query_result(result)
|
||||
check_zulip_realm_query_result(result)
|
||||
MultiuseInvite.objects.all().delete()
|
||||
|
||||
do_send_realm_reactivation_email(get_realm("zulip"))
|
||||
result = self.client_get("/activity/support", {"q": "zulip"})
|
||||
check_realm_reactivation_link_query_result(result)
|
||||
check_zulip_realm_query_result(result)
|
||||
|
||||
def test_change_plan_type(self) -> None:
|
||||
cordelia = self.example_user("cordelia")
|
||||
self.login(cordelia.email)
|
||||
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (cordelia.realm_id,), "plan_type": "2"})
|
||||
self.assertEqual(result.status_code, 302)
|
||||
self.assertEqual(result["Location"], "/login/")
|
||||
|
||||
iago = self.example_user("iago")
|
||||
self.login(iago.email)
|
||||
|
||||
with mock.patch("analytics.views.do_change_plan_type") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (iago.realm_id,), "plan_type": "2"})
|
||||
m.assert_called_once_with(get_realm("zulip"), 2)
|
||||
self.assert_in_success_response(["Plan type of Zulip Dev changed from self hosted to limited"], result)
|
||||
|
||||
def test_attach_discount(self) -> None:
|
||||
lear_realm = get_realm("lear")
|
||||
cordelia_email = self.example_email("cordelia")
|
||||
self.login(cordelia_email)
|
||||
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
|
||||
self.assertEqual(result.status_code, 302)
|
||||
self.assertEqual(result["Location"], "/login/")
|
||||
|
||||
iago_email = self.example_email("iago")
|
||||
self.login(iago_email)
|
||||
|
||||
with mock.patch("analytics.views.attach_discount_to_realm") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
|
||||
m.assert_called_once_with(get_realm("lear"), 25)
|
||||
self.assert_in_success_response(["Discount of Lear & Co. changed to 25 from None"], result)
|
||||
|
||||
def test_activate_or_deactivate_realm(self) -> None:
|
||||
lear_realm = get_realm("lear")
|
||||
cordelia_email = self.example_email("cordelia")
|
||||
self.login(cordelia_email)
|
||||
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "deactivated"})
|
||||
self.assertEqual(result.status_code, 302)
|
||||
self.assertEqual(result["Location"], "/login/")
|
||||
|
||||
iago_email = self.example_email("iago")
|
||||
self.login(iago_email)
|
||||
|
||||
with mock.patch("analytics.views.do_deactivate_realm") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "deactivated"})
|
||||
m.assert_called_once_with(lear_realm, self.example_user("iago"))
|
||||
self.assert_in_success_response(["Lear & Co. deactivated"], result)
|
||||
|
||||
with mock.patch("analytics.views.do_send_realm_reactivation_email") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "status": "active"})
|
||||
m.assert_called_once_with(lear_realm)
|
||||
self.assert_in_success_response(["Realm reactivation email sent to admins of Lear"], result)
|
||||
|
||||
def test_scrub_realm(self) -> None:
|
||||
lear_realm = get_realm("lear")
|
||||
cordelia_email = self.example_email("cordelia")
|
||||
self.login(cordelia_email)
|
||||
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "discount": "25"})
|
||||
self.assertEqual(result.status_code, 302)
|
||||
self.assertEqual(result["Location"], "/login/")
|
||||
|
||||
iago_email = self.example_email("iago")
|
||||
self.login(iago_email)
|
||||
|
||||
with mock.patch("analytics.views.do_scrub_realm") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,), "scrub_realm": "scrub_realm"})
|
||||
m.assert_called_once_with(lear_realm)
|
||||
self.assert_in_success_response(["Lear & Co. scrubbed"], result)
|
||||
|
||||
with mock.patch("analytics.views.do_scrub_realm") as m:
|
||||
result = self.client_post("/activity/support", {"realm_id": "%s" % (lear_realm.id,)})
|
||||
m.assert_not_called()
|
||||
|
||||
class TestGetChartDataHelpers(ZulipTestCase):
|
||||
# last_successful_fill is in analytics/models.py, but get_chart_data is
|
||||
# the only function that uses it at the moment
|
||||
def test_last_successful_fill(self) -> None:
|
||||
self.assertIsNone(last_successful_fill('non-existant'))
|
||||
a_time = datetime(2016, 3, 14, 19).replace(tzinfo=utc)
|
||||
one_hour_before = datetime(2016, 3, 14, 18).replace(tzinfo=utc)
|
||||
fillstate = FillState.objects.create(property='property', end_time=a_time,
|
||||
state=FillState.DONE)
|
||||
self.assertEqual(last_successful_fill('property'), a_time)
|
||||
fillstate.state = FillState.STARTED
|
||||
fillstate.save()
|
||||
self.assertEqual(last_successful_fill('property'), one_hour_before)
|
||||
|
||||
def test_sort_by_totals(self) -> None:
|
||||
empty = [] # type: List[int]
|
||||
value_arrays = {'c': [0, 1], 'a': [9], 'b': [1, 1, 1], 'd': empty}
|
||||
self.assertEqual(sort_by_totals(value_arrays), ['a', 'b', 'c', 'd'])
|
||||
|
||||
def test_sort_client_labels(self) -> None:
|
||||
data = {'everyone': {'a': [16], 'c': [15], 'b': [14], 'e': [13], 'd': [12], 'h': [11]},
|
||||
'user': {'a': [6], 'b': [5], 'd': [4], 'e': [3], 'f': [2], 'g': [1]}}
|
||||
self.assertEqual(sort_client_labels(data), ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
|
||||
|
||||
class TestTimeRange(ZulipTestCase):
|
||||
def test_time_range(self) -> None:
|
||||
HOUR = timedelta(hours=1)
|
||||
DAY = timedelta(days=1)
|
||||
|
||||
a_time = datetime(2016, 3, 14, 22, 59).replace(tzinfo=utc)
|
||||
floor_hour = datetime(2016, 3, 14, 22).replace(tzinfo=utc)
|
||||
floor_day = datetime(2016, 3, 14).replace(tzinfo=utc)
|
||||
|
||||
# test start == end
|
||||
self.assertEqual(time_range(a_time, a_time, CountStat.HOUR, None), [])
|
||||
self.assertEqual(time_range(a_time, a_time, CountStat.DAY, None), [])
|
||||
# test start == end == boundary, and min_length == 0
|
||||
self.assertEqual(time_range(floor_hour, floor_hour, CountStat.HOUR, 0), [floor_hour])
|
||||
self.assertEqual(time_range(floor_day, floor_day, CountStat.DAY, 0), [floor_day])
|
||||
# test start and end on different boundaries
|
||||
self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, None),
|
||||
[floor_hour, floor_hour+HOUR])
|
||||
self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, None),
|
||||
[floor_day, floor_day+DAY])
|
||||
# test min_length
|
||||
self.assertEqual(time_range(floor_hour, floor_hour+HOUR, CountStat.HOUR, 4),
|
||||
[floor_hour-2*HOUR, floor_hour-HOUR, floor_hour, floor_hour+HOUR])
|
||||
self.assertEqual(time_range(floor_day, floor_day+DAY, CountStat.DAY, 4),
|
||||
[floor_day-2*DAY, floor_day-DAY, floor_day, floor_day+DAY])
|
||||
|
||||
class TestMapArrays(ZulipTestCase):
|
||||
def test_map_arrays(self) -> None:
|
||||
a = {'desktop app 1.0': [1, 2, 3],
|
||||
'desktop app 2.0': [10, 12, 13],
|
||||
'desktop app 3.0': [21, 22, 23],
|
||||
'website': [1, 2, 3],
|
||||
'ZulipiOS': [1, 2, 3],
|
||||
'ZulipElectron': [2, 5, 7],
|
||||
'ZulipMobile': [1, 5, 7],
|
||||
'ZulipPython': [1, 2, 3],
|
||||
'API: Python': [1, 2, 3],
|
||||
'SomethingRandom': [4, 5, 6],
|
||||
'ZulipGitHubWebhook': [7, 7, 9],
|
||||
'ZulipAndroid': [64, 63, 65]}
|
||||
result = rewrite_client_arrays(a)
|
||||
self.assertEqual(result,
|
||||
{'Old desktop app': [32, 36, 39],
|
||||
'Old iOS app': [1, 2, 3],
|
||||
'Desktop app': [2, 5, 7],
|
||||
'Mobile app': [1, 5, 7],
|
||||
'Website': [1, 2, 3],
|
||||
'Python API': [2, 4, 6],
|
||||
'SomethingRandom': [4, 5, 6],
|
||||
'GitHub webhook': [7, 7, 9],
|
||||
'Old Android app': [64, 63, 65]})
|
||||
@@ -1,61 +1,8 @@
|
||||
from django.conf.urls import include, url
|
||||
from django.conf.urls import patterns, url
|
||||
|
||||
import analytics.views
|
||||
from zerver.lib.rest import rest_dispatch
|
||||
urlpatterns = patterns('analytics.views',
|
||||
url(r'^activity$', 'get_activity'),
|
||||
url(r'^realm_activity/(?P<realm>[\S]+)/$', 'get_realm_activity'),
|
||||
url(r'^user_activity/(?P<email>[\S]+)/$', 'get_user_activity'),
|
||||
)
|
||||
|
||||
i18n_urlpatterns = [
|
||||
# Server admin (user_profile.is_staff) visible stats pages
|
||||
url(r'^activity$', analytics.views.get_activity,
|
||||
name='analytics.views.get_activity'),
|
||||
url(r'^activity/support$', analytics.views.support,
|
||||
name='analytics.views.support'),
|
||||
url(r'^realm_activity/(?P<realm_str>[\S]+)/$', analytics.views.get_realm_activity,
|
||||
name='analytics.views.get_realm_activity'),
|
||||
url(r'^user_activity/(?P<email>[\S]+)/$', analytics.views.get_user_activity,
|
||||
name='analytics.views.get_user_activity'),
|
||||
|
||||
url(r'^stats/realm/(?P<realm_str>[\S]+)/$', analytics.views.stats_for_realm,
|
||||
name='analytics.views.stats_for_realm'),
|
||||
url(r'^stats/installation$', analytics.views.stats_for_installation,
|
||||
name='analytics.views.stats_for_installation'),
|
||||
url(r'^stats/remote/(?P<remote_server_id>[\S]+)/installation$',
|
||||
analytics.views.stats_for_remote_installation,
|
||||
name='analytics.views.stats_for_remote_installation'),
|
||||
url(r'^stats/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)/$',
|
||||
analytics.views.stats_for_remote_realm,
|
||||
name='analytics.views.stats_for_remote_realm'),
|
||||
|
||||
# User-visible stats page
|
||||
url(r'^stats$', analytics.views.stats,
|
||||
name='analytics.views.stats'),
|
||||
]
|
||||
|
||||
# These endpoints are a part of the API (V1), which uses:
|
||||
# * REST verbs
|
||||
# * Basic auth (username:password is email:apiKey)
|
||||
# * Takes and returns json-formatted data
|
||||
#
|
||||
# See rest_dispatch in zerver.lib.rest for an explanation of auth methods used
|
||||
#
|
||||
# All of these paths are accessed by either a /json or /api prefix
|
||||
v1_api_and_json_patterns = [
|
||||
# get data for the graphs at /stats
|
||||
url(r'^analytics/chart_data$', rest_dispatch,
|
||||
{'GET': 'analytics.views.get_chart_data'}),
|
||||
url(r'^analytics/chart_data/realm/(?P<realm_str>[\S]+)$', rest_dispatch,
|
||||
{'GET': 'analytics.views.get_chart_data_for_realm'}),
|
||||
url(r'^analytics/chart_data/installation$', rest_dispatch,
|
||||
{'GET': 'analytics.views.get_chart_data_for_installation'}),
|
||||
url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/installation$', rest_dispatch,
|
||||
{'GET': 'analytics.views.get_chart_data_for_remote_installation'}),
|
||||
url(r'^analytics/chart_data/remote/(?P<remote_server_id>[\S]+)/realm/(?P<remote_realm_id>[\S]+)$',
|
||||
rest_dispatch,
|
||||
{'GET': 'analytics.views.get_chart_data_for_remote_realm'}),
|
||||
]
|
||||
|
||||
i18n_urlpatterns += [
|
||||
url(r'^api/v1/', include(v1_api_and_json_patterns)),
|
||||
url(r'^json/', include(v1_api_and_json_patterns)),
|
||||
]
|
||||
|
||||
urlpatterns = i18n_urlpatterns
|
||||
|
||||
1090
analytics/views.py
1090
analytics/views.py
File diff suppressed because it is too large
Load Diff
10
api/MANIFEST.in
Normal file
10
api/MANIFEST.in
Normal file
@@ -0,0 +1,10 @@
|
||||
recursive-include integrations *
|
||||
include README.md
|
||||
include examples/zuliprc
|
||||
include examples/send-message
|
||||
include examples/subscribe
|
||||
include examples/get-public-streams
|
||||
include examples/unsubscribe
|
||||
include examples/list-members
|
||||
include examples/list-subscriptions
|
||||
include examples/print-messages
|
||||
106
api/README.md
Normal file
106
api/README.md
Normal file
@@ -0,0 +1,106 @@
|
||||
#### Dependencies
|
||||
|
||||
The [Zulip API](https://zulip.com/api) Python bindings require the
|
||||
following Python libraries:
|
||||
|
||||
* simplejson
|
||||
* requests (version >= 0.12.1)
|
||||
|
||||
|
||||
#### Installing
|
||||
|
||||
This package uses distutils, so you can just run:
|
||||
|
||||
python setup.py install
|
||||
|
||||
#### Using the API
|
||||
|
||||
For now, the only fully supported API operation is sending a message.
|
||||
The other API queries work, but are under active development, so
|
||||
please make sure we know you're using them so that we can notify you
|
||||
as we make any changes to them.
|
||||
|
||||
The easiest way to use these API bindings is to base your tools off
|
||||
of the example tools under examples/ in this distribution.
|
||||
|
||||
If you place your API key in the config file `~/.zuliprc` the Python
|
||||
API bindings will automatically read it in. The format of the config
|
||||
file is as follows:
|
||||
|
||||
[api]
|
||||
key=<api key from the web interface>
|
||||
email=<your email address>
|
||||
|
||||
If you are using Zulip Enterprise, you should also add
|
||||
|
||||
site=<your Zulip Enterprise server's URI>
|
||||
|
||||
Alternatively, you may explicitly use "--user" and "--api-key" in our
|
||||
examples, which is especially useful if you are running several bots
|
||||
which share a home directory. There is also a "--site" option for
|
||||
setting the Zulip Enterprise server on the command line.
|
||||
|
||||
You can obtain your Zulip API key, create bots, and manage bots all
|
||||
from your Zulip [settings page](https://zulip.com/#settings).
|
||||
|
||||
A typical simple bot sending API messages will look as follows:
|
||||
|
||||
At the top of the file:
|
||||
|
||||
# Make sure the Zulip API distribution's root directory is in sys.path, then:
|
||||
import zulip
|
||||
zulip_client = zulip.Client(email="your-bot@example.com", client="MyTestClient/0.1")
|
||||
|
||||
When you want to send a message:
|
||||
|
||||
message = {
|
||||
"type": "stream",
|
||||
"to": ["support"],
|
||||
"subject": "your subject",
|
||||
"content": "your content",
|
||||
}
|
||||
zulip_client.send_message(message)
|
||||
|
||||
Additional examples:
|
||||
|
||||
client.send_message({'type': 'stream', 'content': 'Zulip rules!',
|
||||
'subject': 'feedback', 'to': ['support']})
|
||||
client.send_message({'type': 'private', 'content': 'Zulip rules!',
|
||||
'to': ['user1@example.com', 'user2@example.com']})
|
||||
|
||||
send_message() returns a dict guaranteed to contain the following
|
||||
keys: msg, result. For successful calls, result will be "success" and
|
||||
msg will be the empty string. On error, result will be "error" and
|
||||
msg will describe what went wrong.
|
||||
|
||||
#### Logging
|
||||
The Zulip API comes with a ZulipStream class which can be used with the
|
||||
logging module:
|
||||
|
||||
```
|
||||
import zulip
|
||||
import logging
|
||||
stream = zulip.ZulipStream(type="stream", to=["support"], subject="your subject")
|
||||
logger = logging.getLogger("your_logger")
|
||||
logger.addHandler(logging.StreamHandler(stream))
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("This is an INFO test.")
|
||||
logger.debug("This is a DEBUG test.")
|
||||
logger.warn("This is a WARN test.")
|
||||
logger.error("This is a ERROR test.")
|
||||
```
|
||||
|
||||
#### Sending messages
|
||||
|
||||
You can use the included `zulip-send` script to send messages via the
|
||||
API directly from existing scripts.
|
||||
|
||||
zulip-send hamlet@example.com cordelia@example.com -m \
|
||||
"Conscience doth make cowards of us all."
|
||||
|
||||
Alternatively, if you don't want to use your ~/.zuliprc file:
|
||||
|
||||
zulip-send --user shakespeare-bot@example.com \
|
||||
--api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 \
|
||||
hamlet@example.com cordelia@example.com -m \
|
||||
"Conscience doth make cowards of us all."
|
||||
126
api/bin/zulip-send
Executable file
126
api/bin/zulip-send
Executable file
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
# zulip-send -- Sends a message to the specified recipients.
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import logging
|
||||
|
||||
|
||||
logging.basicConfig()
|
||||
|
||||
log = logging.getLogger('zulip-send')
|
||||
|
||||
def do_send_message(client, message_data ):
|
||||
'''Sends a message and optionally prints status about the same.'''
|
||||
|
||||
if message_data['type'] == 'stream':
|
||||
log.info('Sending message to stream "%s", subject "%s"... ' % \
|
||||
(message_data['to'], message_data['subject']))
|
||||
else:
|
||||
log.info('Sending message to %s... ' % message_data['to'])
|
||||
response = client.send_message(message_data)
|
||||
if response['result'] == 'success':
|
||||
log.info('Message sent.')
|
||||
return True
|
||||
else:
|
||||
log.error(response['msg'])
|
||||
return False
|
||||
|
||||
def main(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
|
||||
usage = """%prog [options] [recipient...]
|
||||
|
||||
Sends a message specified recipients.
|
||||
|
||||
Examples: %prog --stream denmark --subject castle -m "Something is rotten in the state of Denmark."
|
||||
%prog hamlet@example.com cordelia@example.com -m "Conscience doth make cowards of us all."
|
||||
|
||||
These examples assume you have a proper '~/.zuliprc'. You may also set your credentials with the
|
||||
'--user' and '--api-key' arguments.
|
||||
"""
|
||||
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
|
||||
# Grab parser options from the API common set
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
|
||||
parser.add_option('-m', '--message',
|
||||
help='Specifies the message to send, prevents interactive prompting.')
|
||||
|
||||
group = optparse.OptionGroup(parser, 'Stream parameters')
|
||||
group.add_option('-s', '--stream',
|
||||
dest='stream',
|
||||
action='store',
|
||||
help='Allows the user to specify a stream for the message.')
|
||||
group.add_option('-S', '--subject',
|
||||
dest='subject',
|
||||
action='store',
|
||||
help='Allows the user to specify a subject for the message.')
|
||||
parser.add_option_group(group)
|
||||
|
||||
|
||||
(options, recipients) = parser.parse_args(argv[1:])
|
||||
|
||||
if options.verbose:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
# Sanity check user data
|
||||
if len(recipients) != 0 and (options.stream or options.subject):
|
||||
parser.error('You cannot specify both a username and a stream/subject.')
|
||||
if len(recipients) == 0 and (bool(options.stream) != bool(options.subject)):
|
||||
parser.error('Stream messages must have a subject')
|
||||
if len(recipients) == 0 and not (options.stream and options.subject):
|
||||
parser.error('You must specify a stream/subject or at least one recipient.')
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
if not options.message:
|
||||
options.message = sys.stdin.read()
|
||||
|
||||
if options.stream:
|
||||
message_data = {
|
||||
'type': 'stream',
|
||||
'content': options.message,
|
||||
'subject': options.subject,
|
||||
'to': options.stream,
|
||||
}
|
||||
else:
|
||||
message_data = {
|
||||
'type': 'private',
|
||||
'content': options.message,
|
||||
'to': recipients,
|
||||
}
|
||||
|
||||
if not do_send_message(client, message_data):
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
54
api/examples/create-user
Executable file
54
api/examples/create-user
Executable file
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
from os import path
|
||||
import optparse
|
||||
|
||||
usage = """create-user --new-email=<email address> --new-password=<password> --new-full-name=<full name> --new-short-name=<short name> [options]
|
||||
|
||||
Create a user. You must be a realm admin to use this API, and the user
|
||||
will be created in your realm.
|
||||
|
||||
Example: create-user --site=http://localhost:9991 --user=rwbarton@zulip.com --new-email=jarthur@zulip.com --new-password=random17 --new-full-name 'J. Arthur Random' --new-short-name='jarthur'
|
||||
"""
|
||||
|
||||
sys.path.append(path.join(path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
parser.add_option('--new-email')
|
||||
parser.add_option('--new-password')
|
||||
parser.add_option('--new-full-name')
|
||||
parser.add_option('--new-short-name')
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
print client.create_user({
|
||||
'email': options.new_email,
|
||||
'password': options.new_password,
|
||||
'full_name': options.new_full_name,
|
||||
'short_name': options.new_short_name
|
||||
})
|
||||
56
api/examples/edit-message
Executable file
56
api/examples/edit-message
Executable file
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """edit-message [options] --message=<msg_id> --subject=<new subject> --content=<new content> --user=<sender's email address> --api-key=<sender's api key>
|
||||
|
||||
Edits a message that you sent
|
||||
|
||||
Example: edit-message --message-id="348135" --subject="my subject" --content="test message" --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option('--message-id', default="")
|
||||
parser.add_option('--subject', default="")
|
||||
parser.add_option('--content', default="")
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
message_data = {
|
||||
"message_id": options.message_id,
|
||||
}
|
||||
if options.subject != "":
|
||||
message_data["subject"] = options.subject
|
||||
if options.content != "":
|
||||
message_data["content"] = options.content
|
||||
print client.update_message(message_data)
|
||||
46
api/examples/get-public-streams
Executable file
46
api/examples/get-public-streams
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """get-public-streams --user=<bot's email address> --api-key=<bot's api key> [options]
|
||||
|
||||
Prints out all the public streams in the realm.
|
||||
|
||||
Example: get-public-streams --user=othello-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
print client.get_streams(include_public=True, include_subscribed=False)
|
||||
45
api/examples/list-members
Executable file
45
api/examples/list-members
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """list-members --user=<bot's email address> --api-key=<bot's api key> [options]
|
||||
|
||||
List the names and e-mail addresses of the people in your realm.
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
for user in client.get_members()["members"]:
|
||||
print user["full_name"], user["email"]
|
||||
45
api/examples/list-subscriptions
Executable file
45
api/examples/list-subscriptions
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """list-subscriptions --user=<bot's email address> --api-key=<bot's api key> [options]
|
||||
|
||||
Prints out a list of the user's subscriptions.
|
||||
|
||||
Example: list-subscriptions --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
print client.list_subscriptions()
|
||||
49
api/examples/print-messages
Executable file
49
api/examples/print-messages
Executable file
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """print-messages --user=<bot's email address> --api-key=<bot's api key> [options]
|
||||
|
||||
Prints out each message received by the indicated bot or user.
|
||||
|
||||
Example: print-messages --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
def print_message(message):
|
||||
print message
|
||||
|
||||
# This is a blocking call, and will continuously poll for new messages
|
||||
client.call_on_each_message(print_message)
|
||||
45
api/examples/print-next-message
Executable file
45
api/examples/print-next-message
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """print-next-message --user=<bot's email address> --api-key=<bot's api key> [options]
|
||||
|
||||
Prints out the next message received by the user.
|
||||
|
||||
Example: print-next-messages --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
print client.get_messages({})
|
||||
57
api/examples/send-message
Executable file
57
api/examples/send-message
Executable file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
usage = """send-message --user=<bot's email address> --api-key=<bot's api key> [options] <recipients>
|
||||
|
||||
Sends a test message to the specified recipients.
|
||||
|
||||
Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --type=stream commits --subject="my subject" --message="test message"
|
||||
Example: send-message --user=your-bot@example.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 user1@example.com user2@example.com
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option('--subject', default="test")
|
||||
parser.add_option('--message', default="test message")
|
||||
parser.add_option('--type', default='private')
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if len(args) == 0:
|
||||
parser.error("You must specify recipients")
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
message_data = {
|
||||
"type": options.type,
|
||||
"content": options.message,
|
||||
"subject": options.subject,
|
||||
"to": args,
|
||||
}
|
||||
print client.send_message(message_data)
|
||||
52
api/examples/subscribe
Executable file
52
api/examples/subscribe
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """subscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
|
||||
|
||||
Ensures the user is subscribed to the listed streams.
|
||||
|
||||
Examples: subscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
|
||||
subscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
parser.add_option('--streams', default='')
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
if options.streams == "":
|
||||
print >>sys.stderr, "Usage:", parser.usage
|
||||
sys.exit(1)
|
||||
|
||||
print client.add_subscriptions([{"name": stream_name} for stream_name in
|
||||
options.streams.split()])
|
||||
51
api/examples/unsubscribe
Executable file
51
api/examples/unsubscribe
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
|
||||
usage = """unsubscribe --user=<bot's email address> --api-key=<bot's api key> [options] --streams=<streams>
|
||||
|
||||
Ensures the user is not subscribed to the listed streams.
|
||||
|
||||
Examples: unsubscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams=foo
|
||||
unsubscribe --user=tabbott@zulip.com --api-key=a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --streams='foo bar'
|
||||
|
||||
You can omit --user and --api-key arguments if you have a properly set up ~/.zuliprc
|
||||
"""
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
|
||||
import zulip
|
||||
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
parser.add_option('--streams', default='')
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
client = zulip.init_from_options(options)
|
||||
|
||||
if options.streams == "":
|
||||
print >>sys.stderr, "Usage:", parser.usage
|
||||
sys.exit(1)
|
||||
|
||||
print client.remove_subscriptions(options.streams.split())
|
||||
4
api/examples/zuliprc
Normal file
4
api/examples/zuliprc
Normal file
@@ -0,0 +1,4 @@
|
||||
; Save this file as ~/.zuliprc
|
||||
[api]
|
||||
key=<your bot's api key from the web interface>
|
||||
email=<your bot's email address>
|
||||
57
api/integrations/asana/zulip_asana_config.py
Normal file
57
api/integrations/asana/zulip_asana_config.py
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
### REQUIRED CONFIGURATION ###
|
||||
|
||||
# Change these values to your Asana credentials.
|
||||
ASANA_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# Change these values to the credentials for your Asana bot.
|
||||
ZULIP_USER = "asana-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# The Zulip stream that will receive Asana task updates.
|
||||
ZULIP_STREAM_NAME = "asana"
|
||||
|
||||
|
||||
### OPTIONAL CONFIGURATION ###
|
||||
|
||||
# Set to None for logging to stdout when testing, and to a file for
|
||||
# logging when deployed.
|
||||
#LOG_FILE = "/var/tmp/zulip_asana.log"
|
||||
LOG_FILE = None
|
||||
|
||||
# This file is used to resume this mirror in case the script shuts down.
|
||||
# It is required and needs to be writeable.
|
||||
RESUME_FILE = "/var/tmp/zulip_asana.state"
|
||||
|
||||
# When initially started, how many hours of messages to include.
|
||||
ASANA_INITIAL_HISTORY_HOURS = 1
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
|
||||
# If properly installed, the Zulip API should be in your import
|
||||
# path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
277
api/integrations/asana/zulip_asana_mirror
Normal file
277
api/integrations/asana/zulip_asana_mirror
Normal file
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Asana integration for Zulip
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import base64
|
||||
from datetime import datetime, timedelta
|
||||
import dateutil.parser
|
||||
import dateutil.tz
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import urllib2
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_asana_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
import zulip
|
||||
|
||||
if config.LOG_FILE:
|
||||
logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
client = zulip.Client(email=config.ZULIP_USER, api_key=config.ZULIP_API_KEY,
|
||||
site=config.ZULIP_SITE, client="ZulipAsana/" + VERSION)
|
||||
|
||||
def fetch_from_asana(path):
|
||||
"""
|
||||
Request a resource through the Asana API, authenticating using
|
||||
HTTP basic auth.
|
||||
"""
|
||||
auth = base64.encodestring('%s:' % (config.ASANA_API_KEY,))
|
||||
headers = {"Authorization": "Basic %s" % auth}
|
||||
|
||||
url = "https://app.asana.com/api/1.0" + path
|
||||
request = urllib2.Request(url, None, headers)
|
||||
result = urllib2.urlopen(request)
|
||||
|
||||
return json.load(result)
|
||||
|
||||
def send_zulip(topic, content):
|
||||
"""
|
||||
Send a message to Zulip using the configured stream and bot credentials.
|
||||
"""
|
||||
message = {"type": "stream",
|
||||
"sender": config.ZULIP_USER,
|
||||
"to": config.ZULIP_STREAM_NAME,
|
||||
"subject": topic,
|
||||
"content": content,
|
||||
}
|
||||
return client.send_message(message)
|
||||
|
||||
def datestring_to_datetime(datestring):
|
||||
"""
|
||||
Given an ISO 8601 datestring, return the corresponding datetime object.
|
||||
"""
|
||||
return dateutil.parser.parse(datestring).replace(
|
||||
tzinfo=dateutil.tz.gettz('Z'))
|
||||
|
||||
class TaskDict(dict):
|
||||
"""
|
||||
A helper class to turn a dictionary with task information into an
|
||||
object where each of the keys is an attribute for easy access.
|
||||
"""
|
||||
def __getattr__(self, field):
|
||||
return self.get(field)
|
||||
|
||||
def format_topic(task, projects):
|
||||
"""
|
||||
Return a string that will be the Zulip message topic for this task.
|
||||
"""
|
||||
# Tasks can be associated with multiple projects, but in practice they seem
|
||||
# to mostly be associated with one.
|
||||
project_name = projects[task.projects[0]["id"]]
|
||||
return "%s: %s" % (project_name, task.name)
|
||||
|
||||
def format_assignee(task, users):
|
||||
"""
|
||||
Return a string describing the task's assignee.
|
||||
"""
|
||||
if task.assignee:
|
||||
assignee_name = users[task.assignee["id"]]
|
||||
assignee_info = "**Assigned to**: %s (%s)" % (
|
||||
assignee_name, task.assignee_status)
|
||||
else:
|
||||
assignee_info = "**Status**: Unassigned"
|
||||
|
||||
return assignee_info
|
||||
|
||||
def format_due_date(task):
|
||||
"""
|
||||
Return a string describing the task's due date.
|
||||
"""
|
||||
if task.due_on:
|
||||
due_date_info = "**Due on**: %s" % (task.due_on,)
|
||||
else:
|
||||
due_date_info = "**Due date**: None"
|
||||
return due_date_info
|
||||
|
||||
def format_task_creation_event(task, projects, users):
|
||||
"""
|
||||
Format the topic and content for a newly-created task.
|
||||
"""
|
||||
topic = format_topic(task, projects)
|
||||
assignee_info = format_assignee(task, users)
|
||||
due_date_info = format_due_date(task)
|
||||
|
||||
content = """Task **%s** created:
|
||||
|
||||
~~~ quote
|
||||
%s
|
||||
~~~
|
||||
|
||||
%s
|
||||
%s
|
||||
""" % (task.name, task.notes, assignee_info, due_date_info)
|
||||
return topic, content
|
||||
|
||||
def format_task_completion_event(task, projects, users):
|
||||
"""
|
||||
Format the topic and content for a completed task.
|
||||
"""
|
||||
topic = format_topic(task, projects)
|
||||
assignee_info = format_assignee(task, users)
|
||||
due_date_info = format_due_date(task)
|
||||
|
||||
content = """Task **%s** completed. :white_check_mark:
|
||||
|
||||
%s
|
||||
%s
|
||||
""" % (task.name, assignee_info, due_date_info)
|
||||
return topic, content
|
||||
|
||||
def since():
|
||||
"""
|
||||
Return a newness threshold for task events to be processed.
|
||||
"""
|
||||
# If we have a record of the last event processed and it is recent, use it,
|
||||
# else process everything from ASANA_INITIAL_HISTORY_HOURS ago.
|
||||
def default_since():
|
||||
return datetime.utcnow() - timedelta(
|
||||
hours=config.ASANA_INITIAL_HISTORY_HOURS)
|
||||
|
||||
if os.path.exists(config.RESUME_FILE):
|
||||
try:
|
||||
with open(config.RESUME_FILE, "r") as f:
|
||||
datestring = f.readline().strip()
|
||||
timestamp = float(datestring)
|
||||
max_timestamp_processed = datetime.fromtimestamp(timestamp)
|
||||
logging.info("Reading from resume file: " + datestring)
|
||||
except (ValueError,IOError) as e:
|
||||
logging.warn("Could not open resume file: %s" % (
|
||||
e.message or e.strerror,))
|
||||
max_timestamp_processed = default_since()
|
||||
else:
|
||||
logging.info("No resume file, processing an initial history.")
|
||||
max_timestamp_processed = default_since()
|
||||
|
||||
# Even if we can read a timestamp from RESUME_FILE, if it is old don't use
|
||||
# it.
|
||||
return max(max_timestamp_processed, default_since())
|
||||
|
||||
def process_new_events():
|
||||
"""
|
||||
Forward new Asana task events to Zulip.
|
||||
"""
|
||||
# In task queries, Asana only exposes IDs for projects and users, so we need
|
||||
# to look up the mappings.
|
||||
projects = dict((elt["id"], elt["name"]) for elt in \
|
||||
fetch_from_asana("/projects")["data"])
|
||||
users = dict((elt["id"], elt["name"]) for elt in \
|
||||
fetch_from_asana("/users")["data"])
|
||||
|
||||
cutoff = since()
|
||||
max_timestamp_processed = cutoff
|
||||
time_operations = (("created_at", format_task_creation_event),
|
||||
("completed_at", format_task_completion_event))
|
||||
task_fields = ["assignee", "assignee_status", "created_at", "completed_at",
|
||||
"modified_at", "due_on", "name", "notes", "projects"]
|
||||
|
||||
# First, gather all of the tasks that need processing. We'll
|
||||
# process them in order.
|
||||
new_events = []
|
||||
|
||||
for project_id in projects:
|
||||
project_url = "/projects/%d/tasks?opt_fields=%s" % (
|
||||
project_id, ",".join(task_fields))
|
||||
tasks = fetch_from_asana(project_url)["data"]
|
||||
|
||||
for task in tasks:
|
||||
task = TaskDict(task)
|
||||
|
||||
for time_field, operation in time_operations:
|
||||
if task[time_field]:
|
||||
operation_time = datestring_to_datetime(task[time_field])
|
||||
if operation_time > cutoff:
|
||||
new_events.append((operation_time, time_field, operation, task))
|
||||
|
||||
new_events.sort()
|
||||
now = datetime.utcnow()
|
||||
|
||||
for operation_time, time_field, operation, task in new_events:
|
||||
# Unfortunately, creating an Asana task is not an atomic operation. If
|
||||
# the task was just created, or is missing basic information, it is
|
||||
# probably because the task is still being filled out -- wait until the
|
||||
# next round to process it.
|
||||
if (time_field == "created_at") and \
|
||||
(now - operation_time < timedelta(seconds=30)):
|
||||
# The task was just created, give the user some time to fill out
|
||||
# more information.
|
||||
return
|
||||
|
||||
if (time_field == "created_at") and (not task.name) and \
|
||||
(now - operation_time < timedelta(seconds=60)):
|
||||
# If this new task hasn't had a name for a full 30 seconds, assume
|
||||
# you don't plan on giving it one.
|
||||
return
|
||||
|
||||
topic, content = operation(task, projects, users)
|
||||
logging.info("Sending Zulip for " + topic)
|
||||
result = send_zulip(topic, content)
|
||||
|
||||
# If the Zulip wasn't sent successfully, don't update the
|
||||
# max timestamp processed so the task has another change to
|
||||
# be forwarded. Exit, giving temporary issues time to
|
||||
# resolve.
|
||||
if not result.get("result"):
|
||||
logging.warn("Malformed result, exiting:")
|
||||
logging.warn(result)
|
||||
return
|
||||
|
||||
if result["result"] != "success":
|
||||
logging.warn(result["msg"])
|
||||
return
|
||||
|
||||
if operation_time > max_timestamp_processed:
|
||||
max_timestamp_processed = operation_time
|
||||
|
||||
if max_timestamp_processed > cutoff:
|
||||
max_datestring = max_timestamp_processed.strftime("%s.%f")
|
||||
logging.info("Updating resume file: " + max_datestring)
|
||||
open(config.RESUME_FILE, 'w').write(max_datestring)
|
||||
|
||||
while True:
|
||||
try:
|
||||
process_new_events()
|
||||
time.sleep(5)
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Shutting down...")
|
||||
logging.info("Set LOG_FILE to log to a file instead of stdout.")
|
||||
break
|
||||
53
api/integrations/basecamp/zulip_basecamp_config.py
Normal file
53
api/integrations/basecamp/zulip_basecamp_config.py
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
|
||||
# Change these values to configure authentication for basecamp account
|
||||
BASECAMP_ACCOUNT_ID = "12345678"
|
||||
BASECAMP_USERNAME = "foo@example.com"
|
||||
BASECAMP_PASSWORD = "p455w0rd"
|
||||
|
||||
# This script will mirror this many hours of history on the first run.
|
||||
# On subsequent runs this value is ignored.
|
||||
BASECAMP_INITIAL_HISTORY_HOURS = 0
|
||||
|
||||
# Change these values to configure Zulip authentication for the plugin
|
||||
ZULIP_USER = "basecamp-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
ZULIP_STREAM_NAME = "basecamp"
|
||||
|
||||
## If properly installed, the Zulip API should be in your import
|
||||
## path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
|
||||
# If you wish to log to a file rather than stdout/stderr,
|
||||
# please fill this out your desired path
|
||||
LOG_FILE = None
|
||||
|
||||
# This file is used to resume this mirror in case the script shuts down.
|
||||
# It is required and needs to be writeable.
|
||||
RESUME_FILE = "/var/tmp/zulip_basecamp.state"
|
||||
180
api/integrations/basecamp/zulip_basecamp_mirror
Executable file
180
api/integrations/basecamp/zulip_basecamp_mirror
Executable file
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Zulip mirror of Basecamp activity
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
# The "basecamp-mirror.py" script is run continuously, possibly on a work computer
|
||||
# or preferably on a server.
|
||||
# You may need to install the python-requests library.
|
||||
|
||||
import requests
|
||||
import logging
|
||||
import time
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from HTMLParser import HTMLParser
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_basecamp_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
import zulip
|
||||
|
||||
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipBasecamp/" + VERSION)
|
||||
user_agent = "Basecamp To Zulip Mirroring script (support@zulip.com)"
|
||||
htmlParser = HTMLParser()
|
||||
|
||||
# find some form of JSON loader/dumper, with a preference order for speed.
|
||||
json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
|
||||
|
||||
while len(json_implementations):
|
||||
try:
|
||||
json = __import__(json_implementations.pop(0))
|
||||
break
|
||||
except ImportError:
|
||||
continue
|
||||
|
||||
# void function that checks the permissions of the files this script needs.
|
||||
def check_permissions():
|
||||
# check that the log file can be written
|
||||
if config.LOG_FILE:
|
||||
try:
|
||||
open(config.LOG_FILE, "w")
|
||||
except IOError as e:
|
||||
sys.stderr("Could not open up log for writing:")
|
||||
sys.stderr(e)
|
||||
# check that the resume file can be written (this creates if it doesn't exist)
|
||||
try:
|
||||
open(config.RESUME_FILE, "a+")
|
||||
except IOError as e:
|
||||
sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
|
||||
sys.stderr(e)
|
||||
|
||||
# builds the message dict for sending a message with the Zulip API
|
||||
def build_message(event):
|
||||
if not (event.has_key('bucket') and event.has_key('creator') and event.has_key('html_url')):
|
||||
logging.error("Perhaps the Basecamp API changed behavior? "
|
||||
"This event doesn't have the expected format:\n%s" %(event,))
|
||||
return None
|
||||
# adjust the topic length to be bounded to 60 characters
|
||||
topic = event['bucket']['name']
|
||||
if len(topic) > 60:
|
||||
topic = topic[0:57] + "..."
|
||||
# get the action and target values
|
||||
action = htmlParser.unescape(re.sub(r"<[^<>]+>", "", event.get('action', '')))
|
||||
target = htmlParser.unescape(event.get('target', ''))
|
||||
# Some events have "excerpts", which we blockquote
|
||||
excerpt = htmlParser.unescape(event.get('excerpt',''))
|
||||
if excerpt.strip() == "":
|
||||
message = '**%s** %s [%s](%s).' % (event['creator']['name'], action, target, event['html_url'])
|
||||
else:
|
||||
message = '**%s** %s [%s](%s).\n> %s' % (event['creator']['name'], action, target, event['html_url'], excerpt)
|
||||
# assemble the message data dict
|
||||
message_data = {
|
||||
"type": "stream",
|
||||
"to": config.ZULIP_STREAM_NAME,
|
||||
"subject": topic,
|
||||
"content": message,
|
||||
}
|
||||
return message_data
|
||||
|
||||
# the main run loop for this mirror script
|
||||
def run_mirror():
|
||||
# we should have the right (write) permissions on the resume file, as seen
|
||||
# in check_permissions, but it may still be empty or corrupted
|
||||
try:
|
||||
with open(config.RESUME_FILE) as f:
|
||||
since = f.read()
|
||||
since = re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}-\d{2}:\d{2}", since)
|
||||
assert since, "resume file does not meet expected format"
|
||||
since = since.string
|
||||
except (AssertionError,IOError) as e:
|
||||
logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
|
||||
since = (datetime.utcnow() - timedelta(hours=config.BASECAMP_INITIAL_HISTORY_HOURS)).isoformat() + "-00:00"
|
||||
try:
|
||||
# we use an exponential backoff approach when we get 429 (Too Many Requests).
|
||||
sleepInterval = 1
|
||||
while 1:
|
||||
time.sleep(sleepInterval)
|
||||
response = requests.get("https://basecamp.com/%s/api/v1/events.json" % (config.BASECAMP_ACCOUNT_ID),
|
||||
params={'since': since},
|
||||
auth=(config.BASECAMP_USERNAME, config.BASECAMP_PASSWORD),
|
||||
headers = {"User-Agent": user_agent})
|
||||
if response.status_code == 200:
|
||||
sleepInterval = 1
|
||||
events = json.loads(response.text)
|
||||
if len(events):
|
||||
logging.info("Got event(s): %s" % (response.text,))
|
||||
if response.status_code >= 500:
|
||||
logging.error(response.status_code)
|
||||
continue
|
||||
if response.status_code == 429:
|
||||
# exponential backoff
|
||||
sleepInterval *= 2
|
||||
logging.error(response.status_code)
|
||||
continue
|
||||
if response.status_code == 400:
|
||||
logging.error("Something went wrong. Basecamp must be unhappy for this reason: %s" % (response.text,))
|
||||
sys.exit(-1)
|
||||
if response.status_code == 401:
|
||||
logging.error("Bad authorization from Basecamp. Please check your Basecamp login credentials")
|
||||
sys.exit(-1)
|
||||
if len(events):
|
||||
since = events[0]['created_at']
|
||||
for event in reversed(events):
|
||||
message_data = build_message(event)
|
||||
if not message_data:
|
||||
continue
|
||||
zulip_api_result = client.send_message(message_data)
|
||||
if zulip_api_result['result'] == "success":
|
||||
logging.info("sent zulip with id: %s" % (zulip_api_result['id'],))
|
||||
else:
|
||||
logging.warn("%s %s" % (zulip_api_result['result'], zulip_api_result['msg']))
|
||||
# update 'since' each time in case we get KeyboardInterrupted
|
||||
since = event['created_at']
|
||||
# avoid hitting rate-limit
|
||||
time.sleep(0.2)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Shutting down, please hold")
|
||||
open("events.last", 'w').write(since)
|
||||
logging.info("Done!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not isinstance(config.RESUME_FILE, basestring):
|
||||
sys.stderr("RESUME_FILE path not given; refusing to continue")
|
||||
check_permissions()
|
||||
if config.LOG_FILE:
|
||||
logging.basicConfig(filename=config.LOG_FILE, level=logging.INFO)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
run_mirror()
|
||||
62
api/integrations/codebase/zulip_codebase_config.py
Normal file
62
api/integrations/codebase/zulip_codebase_config.py
Normal file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
|
||||
# Change these values to configure authentication for your codebase account
|
||||
# Note that this is the Codebase API Username, found in the Settings page
|
||||
# for your account
|
||||
CODEBASE_API_USERNAME = "foo@example.com"
|
||||
CODEBASE_API_KEY = "1234561234567abcdef"
|
||||
|
||||
# The URL of your codebase setup
|
||||
CODEBASE_ROOT_URL = "https://YOUR_COMPANY.codebasehq.com"
|
||||
|
||||
# When initially started, how many hours of messages to include.
|
||||
# Note that the Codebase API only returns the 20 latest events,
|
||||
# if you have more than 20 events that fit within this window,
|
||||
# earlier ones may be lost
|
||||
CODEBASE_INITIAL_HISTORY_HOURS = 12
|
||||
|
||||
# Change these values to configure Zulip authentication for the plugin
|
||||
ZULIP_USER = "codebase-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# The streams to send commit information and ticket information to
|
||||
ZULIP_COMMITS_STREAM_NAME = "codebase"
|
||||
ZULIP_TICKETS_STREAM_NAME = "tickets"
|
||||
|
||||
# If properly installed, the Zulip API should be in your import
|
||||
# path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
|
||||
# If you wish to log to a file rather than stdout/stderr,
|
||||
# please fill this out your desired path
|
||||
LOG_FILE = None
|
||||
|
||||
# This file is used to resume this mirror in case the script shuts down.
|
||||
# It is required and needs to be writeable.
|
||||
RESUME_FILE = "/var/tmp/zulip_codebase.state"
|
||||
318
api/integrations/codebase/zulip_codebase_mirror
Executable file
318
api/integrations/codebase/zulip_codebase_mirror
Executable file
@@ -0,0 +1,318 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Zulip mirror of Codebase HQ activity
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
# The "codebase-mirror.py" script is run continuously, possibly on a work computer
|
||||
# or preferably on a server.
|
||||
#
|
||||
# When restarted, it will attempt to pick up where it left off.
|
||||
#
|
||||
# You may need to install the python-requests library, as well as python-dateutil
|
||||
|
||||
import requests
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_codebase_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
import zulip
|
||||
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipCodebase/" + VERSION)
|
||||
user_agent = "Codebase To Zulip Mirroring script (support@zulip.com)"
|
||||
|
||||
# find some form of JSON loader/dumper, with a preference order for speed.
|
||||
json_implementations = ['ujson', 'cjson', 'simplejson', 'json']
|
||||
|
||||
while len(json_implementations):
|
||||
try:
|
||||
json = __import__(json_implementations.pop(0))
|
||||
break
|
||||
except ImportError:
|
||||
continue
|
||||
|
||||
def make_api_call(path):
|
||||
response = requests.get("https://api3.codebasehq.com/%s" % (path,),
|
||||
auth=(config.CODEBASE_API_USERNAME, config.CODEBASE_API_KEY),
|
||||
params={'raw': True},
|
||||
headers = {"User-Agent": user_agent,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json"})
|
||||
if response.status_code == 200:
|
||||
return json.loads(response.text)
|
||||
|
||||
if response.status_code >= 500:
|
||||
logging.error(response.status_code)
|
||||
return None
|
||||
if response.status_code == 403:
|
||||
logging.error("Bad authorization from Codebase. Please check your credentials")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
logging.warn("Found non-success response status code: %s %s" % (response.status_code, response.text))
|
||||
return None
|
||||
|
||||
def make_url(path):
|
||||
return "%s/%s" % (config.CODEBASE_ROOT_URL, path)
|
||||
|
||||
def handle_event(event):
|
||||
event = event['event']
|
||||
event_type = event['type']
|
||||
actor_name = event['actor_name']
|
||||
|
||||
raw_props = event.get('raw_properties', {})
|
||||
|
||||
project_link = raw_props.get('project_permalink')
|
||||
|
||||
subject = None
|
||||
content = None
|
||||
if event_type == 'repository_creation':
|
||||
stream = config.ZULIP_COMMITS_STREAM_NAME
|
||||
|
||||
project_name = raw_props.get('name')
|
||||
project_repo_type = raw_props.get('scm_type')
|
||||
|
||||
url = make_url("projects/%s" % project_link)
|
||||
scm = "of type %s" % (project_repo_type,) if project_repo_type else ""
|
||||
|
||||
|
||||
subject = "Repository %s Created" % (project_name,)
|
||||
content = "%s created a new repository %s [%s](%s)" % (actor_name, scm, project_name, url)
|
||||
elif event_type == 'push':
|
||||
stream = config.ZULIP_COMMITS_STREAM_NAME
|
||||
|
||||
num_commits = raw_props.get('commits_count')
|
||||
branch = raw_props.get('ref_name')
|
||||
project = raw_props.get('project_name')
|
||||
repo_link = raw_props.get('repository_permalink')
|
||||
deleted_ref = raw_props.get('deleted_ref')
|
||||
new_ref = raw_props.get('new_ref')
|
||||
|
||||
subject = "Push to %s on %s" % (branch, project)
|
||||
|
||||
if deleted_ref:
|
||||
content = "%s deleted branch %s from %s" % (actor_name, branch, project)
|
||||
else:
|
||||
if new_ref:
|
||||
branch = "new branch %s" % (branch, )
|
||||
content = "%s pushed %s commit(s) to %s in project %s:\n\n" % \
|
||||
(actor_name, num_commits, branch, project)
|
||||
for commit in raw_props.get('commits'):
|
||||
ref = commit.get('ref')
|
||||
url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, ref))
|
||||
message = commit.get('message')
|
||||
content += "* [%s](%s): %s\n" % (ref, url, message)
|
||||
elif event_type == 'ticketing_ticket':
|
||||
stream = config.ZULIP_TICKETS_STREAM_NAME
|
||||
|
||||
num = raw_props.get('number')
|
||||
name = raw_props.get('subject')
|
||||
assignee = raw_props.get('assignee')
|
||||
priority = raw_props.get('priority')
|
||||
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
||||
|
||||
if assignee is None:
|
||||
assignee = "no one"
|
||||
subject = "#%s: %s" % (num, name)
|
||||
content = """%s created a new ticket [#%s](%s) priority **%s** assigned to %s:\n\n~~~ quote\n %s""" % \
|
||||
(actor_name, num, url, priority, assignee, name)
|
||||
elif event_type == 'ticketing_note':
|
||||
stream = config.ZULIP_TICKETS_STREAM_NAME
|
||||
|
||||
num = raw_props.get('number')
|
||||
name = raw_props.get('subject')
|
||||
body = raw_props.get('content')
|
||||
changes = raw_props.get('changes')
|
||||
|
||||
|
||||
url = make_url("projects/%s/tickets/%s" % (project_link, num))
|
||||
subject = "#%s: %s" % (num, name)
|
||||
|
||||
content = ""
|
||||
if body is not None and len(body) > 0:
|
||||
content = "%s added a comment to ticket [#%s](%s):\n\n~~~ quote\n%s\n\n" % (actor_name, num, url, body)
|
||||
|
||||
if 'status_id' in changes:
|
||||
status_change = changes.get('status_id')
|
||||
content += "Status changed from **%s** to **%s**\n\n" % (status_change[0], status_change[1])
|
||||
elif event_type == 'ticketing_milestone':
|
||||
stream = config.ZULIP_TICKETS_STREAM_NAME
|
||||
|
||||
name = raw_props.get('name')
|
||||
identifier = raw_props.get('identifier')
|
||||
url = make_url("projects/%s/milestone/%s" % (project_link, identifier))
|
||||
|
||||
subject = name
|
||||
content = "%s created a new milestone [%s](%s)" % (actor_name, name, url)
|
||||
elif event_type == 'comment':
|
||||
stream = config.ZULIP_COMMITS_STREAM_NAME
|
||||
|
||||
comment = raw_props.get('content')
|
||||
commit = raw_props.get('commit_ref')
|
||||
|
||||
# If there's a commit id, it's a comment to a commit
|
||||
if commit:
|
||||
repo_link = raw_props.get('repository_permalink')
|
||||
|
||||
url = make_url('projects/%s/repositories/%s/commit/%s' % (project_link, repo_link, commit))
|
||||
|
||||
subject = "%s commented on %s" % (actor_name, commit)
|
||||
content = "%s commented on [%s](%s):\n\n~~~ quote\n%s" % (actor_name, commit, url, comment)
|
||||
else:
|
||||
# Otherwise, this is a Discussion item, and handle it
|
||||
subj = raw_props.get("subject")
|
||||
category = raw_props.get("category")
|
||||
comment_content = raw_props.get("content")
|
||||
|
||||
subject = "Discussion: %s" % (subj,)
|
||||
|
||||
if category:
|
||||
content = "%s started a new discussion in %s:\n\n~~~ quote\n%s\n~~~" % (actor_name, category, comment_content)
|
||||
else:
|
||||
content = "%s posted:\n\n~~~ quote\n%s\n~~~" % (actor_name, comment_content)
|
||||
|
||||
elif event_type == 'deployment':
|
||||
stream = config.ZULIP_COMMITS_STREAM_NAME
|
||||
|
||||
start_ref = raw_props.get('start_ref')
|
||||
end_ref = raw_props.get('end_ref')
|
||||
environment = raw_props.get('environment')
|
||||
servers = raw_props.get('servers')
|
||||
repo_link = raw_props.get('repository_permalink')
|
||||
|
||||
start_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, start_ref))
|
||||
end_ref_url = make_url("projects/%s/repositories/%s/commit/%s" % (project_link, repo_link, end_ref))
|
||||
between_url = make_url("projects/%s/repositories/%s/compare/%s...%s" % (project_link, repo_link, start_ref, end_ref))
|
||||
|
||||
subject = "Deployment to %s" % (environment,)
|
||||
|
||||
content = "%s deployed [%s](%s) [through](%s) [%s](%s) to the **%s** environment." % \
|
||||
(actor_name, start_ref, start_ref_url, between_url, end_ref, end_ref_url, environment)
|
||||
if servers is not None:
|
||||
content += "\n\nServers deployed to: %s" % (", ".join(["`%s`" % (server,) for server in servers]))
|
||||
|
||||
elif event_type == 'named_tree':
|
||||
# Docs say named_tree type used for new/deleting branches and tags,
|
||||
# but experimental testing showed that they were all sent as 'push' events
|
||||
pass
|
||||
elif event_type == 'wiki_page':
|
||||
logging.warn("Wiki page notifications not yet implemented")
|
||||
elif event_type == 'sprint_creation':
|
||||
logging.warn("Sprint notifications not yet implemented")
|
||||
elif event_type == 'sprint_ended':
|
||||
logging.warn("Sprint notifications not yet implemented")
|
||||
else:
|
||||
logging.info("Unknown event type %s, ignoring!" % (event_type,))
|
||||
|
||||
if subject and content:
|
||||
if len(subject) > 60:
|
||||
subject = subject[:57].rstrip() + '...'
|
||||
|
||||
res = client.send_message({"type": "stream",
|
||||
"to": stream,
|
||||
"subject": subject,
|
||||
"content": content})
|
||||
if res['result'] == 'success':
|
||||
logging.info("Successfully sent Zulip with id: %s" % (res['id']))
|
||||
else:
|
||||
logging.warn("Failed to send Zulip: %s %s" % (res['result'], res['msg']))
|
||||
|
||||
|
||||
# the main run loop for this mirror script
|
||||
def run_mirror():
|
||||
# we should have the right (write) permissions on the resume file, as seen
|
||||
# in check_permissions, but it may still be empty or corrupted
|
||||
def default_since():
|
||||
return datetime.utcnow() - timedelta(hours=config.CODEBASE_INITIAL_HISTORY_HOURS)
|
||||
|
||||
try:
|
||||
with open(config.RESUME_FILE) as f:
|
||||
timestamp = f.read()
|
||||
if timestamp == '':
|
||||
since = default_since()
|
||||
else:
|
||||
timestamp = int(timestamp, 10)
|
||||
since = datetime.fromtimestamp(timestamp)
|
||||
except (ValueError,IOError) as e:
|
||||
logging.warn("Could not open resume file: %s" % (e.message or e.strerror,))
|
||||
since = default_since()
|
||||
|
||||
try:
|
||||
sleepInterval = 1
|
||||
while 1:
|
||||
events = make_api_call("activity")[::-1]
|
||||
if events is not None:
|
||||
sleepInterval = 1
|
||||
for event in events:
|
||||
timestamp = event.get('event', {}).get('timestamp', '')
|
||||
event_date = dateutil.parser.parse(timestamp).replace(tzinfo=None)
|
||||
if event_date > since:
|
||||
handle_event(event)
|
||||
since = event_date
|
||||
else:
|
||||
# back off a bit
|
||||
if sleepInterval < 22:
|
||||
sleepInterval += 4
|
||||
time.sleep(sleepInterval)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
open(config.RESUME_FILE, 'w').write(since.strftime("%s"));
|
||||
logging.info("Shutting down Codebase mirror")
|
||||
|
||||
# void function that checks the permissions of the files this script needs.
|
||||
def check_permissions():
|
||||
# check that the log file can be written
|
||||
if config.LOG_FILE:
|
||||
try:
|
||||
open(config.LOG_FILE, "w")
|
||||
except IOError as e:
|
||||
sys.stderr("Could not open up log for writing:")
|
||||
sys.stderr(e)
|
||||
# check that the resume file can be written (this creates if it doesn't exist)
|
||||
try:
|
||||
open(config.RESUME_FILE, "a+")
|
||||
except IOError as e:
|
||||
sys.stderr("Could not open up the file %s for reading and writing" % (config.RESUME_FILE,))
|
||||
sys.stderr(e)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not isinstance(config.RESUME_FILE, basestring):
|
||||
sys.stderr("RESUME_FILE path not given; refusing to continue")
|
||||
check_permissions()
|
||||
if config.LOG_FILE:
|
||||
logging.basicConfig(filename=config.LOG_FILE, level=logging.WARNING)
|
||||
else:
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
run_mirror()
|
||||
130
api/integrations/git/post-receive
Executable file
130
api/integrations/git/post-receive
Executable file
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Zulip notification post-receive hook.
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
# The "post-receive" script is run after receive-pack has accepted a pack
|
||||
# and the repository has been updated. It is passed arguments in through
|
||||
# stdin in the form
|
||||
# <oldrev> <newrev> <refname>
|
||||
# For example:
|
||||
# aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import os.path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_git_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
|
||||
import zulip
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipGit/" + VERSION)
|
||||
|
||||
# check_output is backported from subprocess.py in Python 2.7
|
||||
def check_output(*popenargs, **kwargs):
|
||||
if 'stdout' in kwargs:
|
||||
raise ValueError('stdout argument not allowed, it will be overridden.')
|
||||
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
|
||||
output, unused_err = process.communicate()
|
||||
retcode = process.poll()
|
||||
if retcode:
|
||||
cmd = kwargs.get("args")
|
||||
if cmd is None:
|
||||
cmd = popenargs[0]
|
||||
raise subprocess.CalledProcessError(retcode, cmd, output=output)
|
||||
return output
|
||||
subprocess.check_output = check_output
|
||||
|
||||
def git_repository_name():
|
||||
output = subprocess.check_output(["git", "rev-parse", "--is-bare-repository"])
|
||||
if output.strip() == "true":
|
||||
return os.path.basename(os.getcwd())[:-len(".git")]
|
||||
else:
|
||||
return os.path.basename(os.path.dirname(os.getcwd()))
|
||||
|
||||
def git_commit_range(oldrev, newrev):
|
||||
log_cmd = ["git", "log", "--reverse",
|
||||
"--pretty=%aE %H %s", "%s..%s" % (oldrev, newrev)]
|
||||
commits = ''
|
||||
for ln in subprocess.check_output(log_cmd).splitlines():
|
||||
author_email, commit_id, subject = ln.split(None, 2)
|
||||
if hasattr(config, "format_commit_message"):
|
||||
commits += config.format_commit_message(author_email, subject, commit_id)
|
||||
else:
|
||||
commits += '!avatar(%s) %s\n' % (author_email, subject)
|
||||
return commits
|
||||
|
||||
def send_bot_message(oldrev, newrev, refname):
|
||||
repo_name = git_repository_name()
|
||||
branch = refname.replace('refs/heads/', '')
|
||||
destination = config.commit_notice_destination(repo_name, branch, newrev)
|
||||
if destination is None:
|
||||
# Don't forward the notice anywhere
|
||||
return
|
||||
|
||||
new_head = newrev[:12]
|
||||
old_head = oldrev[:12]
|
||||
|
||||
if (oldrev == '0000000000000000000000000000000000000000' or
|
||||
newrev == '0000000000000000000000000000000000000000'):
|
||||
# New branch pushed or old branch removed
|
||||
added = ''
|
||||
removed = ''
|
||||
else:
|
||||
added = git_commit_range(oldrev, newrev)
|
||||
removed = git_commit_range(newrev, oldrev)
|
||||
|
||||
if oldrev == '0000000000000000000000000000000000000000':
|
||||
message = '`%s` was pushed to new branch `%s`' % (new_head, branch)
|
||||
elif newrev == '0000000000000000000000000000000000000000':
|
||||
message = 'branch `%s` was removed (was `%s`)' % (branch, old_head)
|
||||
elif removed:
|
||||
message = '`%s` was pushed to `%s`, **REMOVING**:\n\n%s' % (new_head, branch, removed)
|
||||
if added:
|
||||
message += '\n**and adding**:\n\n' + added
|
||||
message += '\n**A HISTORY REWRITE HAS OCCURRED!**'
|
||||
message += '\n@everyone: Please check your local branches to deal with this.'
|
||||
elif added:
|
||||
message = '`%s` was deployed to `%s` with:\n\n%s' % (new_head, branch, added)
|
||||
else:
|
||||
message = '`%s` was pushed to `%s`... but nothing changed?' % (new_head, branch)
|
||||
|
||||
message_data = {
|
||||
"type": "stream",
|
||||
"to": destination["stream"],
|
||||
"subject": destination["subject"],
|
||||
"content": message,
|
||||
}
|
||||
client.send_message(message_data)
|
||||
|
||||
for ln in sys.stdin:
|
||||
oldrev, newrev, refname = ln.strip().split()
|
||||
send_bot_message(oldrev, newrev, refname)
|
||||
65
api/integrations/git/zulip_git_config.py
Normal file
65
api/integrations/git/zulip_git_config.py
Normal file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
# Change these values to configure authentication for the plugin
|
||||
ZULIP_USER = "git-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# commit_notice_destination() lets you customize where commit notices
|
||||
# are sent to with the full power of a Python function.
|
||||
#
|
||||
# It takes the following arguments:
|
||||
# * repo = the name of the git repository
|
||||
# * branch = the name of the branch that was pushed to
|
||||
# * commit = the commit id
|
||||
#
|
||||
# Returns a dictionary encoding the stream and subject to send the
|
||||
# notification to (or None to send no notification).
|
||||
#
|
||||
# The default code below will send every commit pushed to "master" to
|
||||
# * stream "commits"
|
||||
# * topic "master"
|
||||
# And similarly for branch "test-post-receive" (for use when testing).
|
||||
def commit_notice_destination(repo, branch, commit):
|
||||
if branch in ["master", "test-post-receive"]:
|
||||
return dict(stream = "commits",
|
||||
subject = u"%s" % (branch,))
|
||||
|
||||
# Return None for cases where you don't want a notice sent
|
||||
return None
|
||||
|
||||
# Modify this function to change how commits are displayed; the most
|
||||
# common customization is to include a link to the commit in your
|
||||
# graphical repository viewer, e.g.
|
||||
#
|
||||
# return '!avatar(%s) [%s](https://example.com/commits/%s)\n' % (author, subject, commit_id)
|
||||
def format_commit_message(author, subject, commit_id):
|
||||
return '!avatar(%s) %s\n' % (author, subject)
|
||||
|
||||
## If properly installed, the Zulip API should be in your import
|
||||
## path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
170
api/integrations/hg/zulip-changegroup.py
Executable file
170
api/integrations/hg/zulip-changegroup.py
Executable file
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Zulip hook for Mercurial changeset pushes.
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
#
|
||||
# This hook is called when changesets are pushed to the master repository (ie
|
||||
# `hg push`). See https://zulip.com/integrations for installation instructions.
|
||||
|
||||
import zulip
|
||||
|
||||
VERSION = "0.9"
|
||||
|
||||
def format_summary_line(web_url, user, base, tip, branch, node):
|
||||
"""
|
||||
Format the first line of the message, which contains summary
|
||||
information about the changeset and links to the changelog if a
|
||||
web URL has been configured:
|
||||
|
||||
Jane Doe <jane@example.com> pushed 1 commit to master (170:e494a5be3393):
|
||||
"""
|
||||
revcount = tip - base
|
||||
plural = "s" if revcount > 1 else ""
|
||||
|
||||
if web_url:
|
||||
shortlog_base_url = web_url.rstrip("/") + "/shortlog/"
|
||||
summary_url = "{shortlog}{tip}?revcount={revcount}".format(
|
||||
shortlog=shortlog_base_url, tip=tip - 1, revcount=revcount)
|
||||
formatted_commit_count = "[{revcount} commit{s}]({url})".format(
|
||||
revcount=revcount, s=plural, url=summary_url)
|
||||
else:
|
||||
formatted_commit_count = "{revcount} commit{s}".format(
|
||||
revcount=revcount, s=plural)
|
||||
|
||||
return u"**{user}** pushed {commits} to **{branch}** (`{tip}:{node}`):\n\n".format(
|
||||
user=user, commits=formatted_commit_count, branch=branch, tip=tip,
|
||||
node=node[:12])
|
||||
|
||||
def format_commit_lines(web_url, repo, base, tip):
|
||||
"""
|
||||
Format the per-commit information for the message, including the one-line
|
||||
commit summary and a link to the diff if a web URL has been configured:
|
||||
"""
|
||||
if web_url:
|
||||
rev_base_url = web_url.rstrip("/") + "/rev/"
|
||||
|
||||
commit_summaries = []
|
||||
for rev in range(base, tip):
|
||||
rev_node = repo.changelog.node(rev)
|
||||
rev_ctx = repo.changectx(rev_node)
|
||||
one_liner = rev_ctx.description().split("\n")[0]
|
||||
|
||||
if web_url:
|
||||
summary_url = rev_base_url + str(rev_ctx)
|
||||
summary = "* [{summary}]({url})".format(
|
||||
summary=one_liner, url=summary_url)
|
||||
else:
|
||||
summary = "* {summary}".format(summary=one_liner)
|
||||
|
||||
commit_summaries.append(summary)
|
||||
|
||||
return "\n".join(summary for summary in commit_summaries)
|
||||
|
||||
def send_zulip(email, api_key, site, stream, subject, content):
|
||||
"""
|
||||
Send a message to Zulip using the provided credentials, which should be for
|
||||
a bot in most cases.
|
||||
"""
|
||||
client = zulip.Client(email=email, api_key=api_key,
|
||||
site=site,
|
||||
client="ZulipMercurial/" + VERSION)
|
||||
|
||||
message_data = {
|
||||
"type": "stream",
|
||||
"to": stream,
|
||||
"subject": subject,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
client.send_message(message_data)
|
||||
|
||||
def get_config(ui, item):
|
||||
try:
|
||||
# configlist returns everything in lists.
|
||||
return ui.configlist('zulip', item)[0]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def hook(ui, repo, **kwargs):
|
||||
"""
|
||||
Invoked by configuring a [hook] entry in .hg/hgrc.
|
||||
"""
|
||||
hooktype = kwargs["hooktype"]
|
||||
node = kwargs["node"]
|
||||
|
||||
ui.debug("Zulip: received {hooktype} event\n".format(hooktype=hooktype))
|
||||
|
||||
if hooktype != "changegroup":
|
||||
ui.warn("Zulip: {hooktype} not supported\n".format(hooktype=hooktype))
|
||||
exit(1)
|
||||
|
||||
ctx = repo.changectx(node)
|
||||
branch = ctx.branch()
|
||||
|
||||
# If `branches` isn't specified, notify on all branches.
|
||||
branch_whitelist = get_config(ui, "branches")
|
||||
branch_blacklist = get_config(ui, "ignore_branches")
|
||||
|
||||
if branch_whitelist:
|
||||
# Only send notifications on branches we are watching.
|
||||
watched_branches = [b.lower().strip() for b in branch_whitelist.split(",")]
|
||||
if branch.lower() not in watched_branches:
|
||||
ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
|
||||
exit(0)
|
||||
|
||||
if branch_blacklist:
|
||||
# Don't send notifications for branches we've ignored.
|
||||
ignored_branches = [b.lower().strip() for b in branch_blacklist.split(",")]
|
||||
if branch.lower() in ignored_branches:
|
||||
ui.debug("Zulip: ignoring event for {branch}\n".format(branch=branch))
|
||||
exit(0)
|
||||
|
||||
# The first and final commits in the changeset.
|
||||
base = repo[node].rev()
|
||||
tip = len(repo)
|
||||
|
||||
email = get_config(ui, "email")
|
||||
api_key = get_config(ui, "api_key")
|
||||
site = get_config(ui, "site")
|
||||
|
||||
if not (email and api_key):
|
||||
ui.warn("Zulip: missing email or api_key configurations\n")
|
||||
ui.warn("in the [zulip] section of your .hg/hgrc.\n")
|
||||
exit(1)
|
||||
|
||||
stream = get_config(ui, "stream")
|
||||
# Give a default stream if one isn't provided.
|
||||
if not stream:
|
||||
stream = "commits"
|
||||
|
||||
web_url = get_config(ui, "web_url")
|
||||
user = ctx.user()
|
||||
content = format_summary_line(web_url, user, base, tip, branch, node)
|
||||
content += format_commit_lines(web_url, repo, base, tip)
|
||||
|
||||
subject = branch
|
||||
|
||||
ui.debug("Sending to Zulip:\n")
|
||||
ui.debug(content + "\n")
|
||||
|
||||
send_zulip(email, api_key, site, stream, subject, content)
|
||||
149
api/integrations/jira/org/humbug/jira/ZulipListener.groovy
Normal file
149
api/integrations/jira/org/humbug/jira/ZulipListener.groovy
Normal file
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright (c) 2014 Zulip, Inc
|
||||
*/
|
||||
|
||||
package org.zulip.jira
|
||||
|
||||
import static com.atlassian.jira.event.type.EventType.*
|
||||
|
||||
import com.atlassian.jira.event.issue.AbstractIssueEventListener
|
||||
import com.atlassian.jira.event.issue.IssueEvent
|
||||
|
||||
import java.util.logging.Level
|
||||
import java.util.logging.Logger
|
||||
|
||||
import org.apache.commons.httpclient.HttpClient
|
||||
import org.apache.commons.httpclient.HttpStatus;
|
||||
import org.apache.commons.httpclient.methods.PostMethod
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
|
||||
class ZulipListener extends AbstractIssueEventListener {
|
||||
Logger LOGGER = Logger.getLogger(ZulipListener.class.getName());
|
||||
|
||||
// The email address of one of the bots you created on your Zulip settings page.
|
||||
String zulipEmail = ""
|
||||
// That bot's API key.
|
||||
String zulipAPIKey = ""
|
||||
|
||||
// What stream to send messages to. Must already exist.
|
||||
String zulipStream = "jira"
|
||||
|
||||
// The base JIRA url for browsing
|
||||
String issueBaseUrl = "https://jira.COMPANY.com/browse/"
|
||||
|
||||
// Your zulip domain, only change if you have a custom one
|
||||
String base_url = "https://api.zulip.com"
|
||||
|
||||
@Override
|
||||
void workflowEvent(IssueEvent event) {
|
||||
processIssueEvent(event)
|
||||
}
|
||||
|
||||
String processIssueEvent(IssueEvent event) {
|
||||
String author = event.user.displayName
|
||||
String issueId = event.issue.key
|
||||
String issueUrl = issueBaseUrl + issueId
|
||||
String issueUrlMd = String.format("[%s](%s)", issueId, issueBaseUrl + issueId)
|
||||
String title = event.issue.summary
|
||||
String subject = truncate(String.format("%s: %s", issueId, title), 60)
|
||||
String assignee = "no one"
|
||||
if (event.issue.assignee) {
|
||||
assignee = event.issue.assignee.name
|
||||
}
|
||||
String comment = "";
|
||||
if (event.comment) {
|
||||
comment = event.comment.body
|
||||
}
|
||||
|
||||
String content;
|
||||
|
||||
// Event types:
|
||||
// https://docs.atlassian.com/jira/5.0/com/atlassian/jira/event/type/EventType.html
|
||||
// Issue API:
|
||||
// https://docs.atlassian.com/jira/5.0/com/atlassian/jira/issue/Issue.html
|
||||
switch (event.getEventTypeId()) {
|
||||
case ISSUE_COMMENTED_ID:
|
||||
content = String.format("%s **updated** %s with comment:\n\n> %s",
|
||||
author, issueUrlMd, comment)
|
||||
break
|
||||
case ISSUE_CREATED_ID:
|
||||
content = String.format("%s **created** %s priority %s, assigned to **%s**: \n\n> %s",
|
||||
author, issueUrlMd, event.issue.priorityObject.name,
|
||||
assignee, title)
|
||||
break
|
||||
case ISSUE_ASSIGNED_ID:
|
||||
content = String.format("%s **reassigned** %s to **%s**",
|
||||
author, issueUrlMd, assignee)
|
||||
break
|
||||
case ISSUE_DELETED_ID:
|
||||
content = String.format("%s **deleted** %s!",
|
||||
author, issueUrlMd)
|
||||
break
|
||||
case ISSUE_RESOLVED_ID:
|
||||
content = String.format("%s **resolved** %s as %s:\n\n> %s",
|
||||
author, issueUrlMd, event.issue.resolutionObject.name,
|
||||
comment)
|
||||
break
|
||||
case ISSUE_CLOSED_ID:
|
||||
content = String.format("%s **closed** %s with resolution %s:\n\n> %s",
|
||||
author, issueUrlMd, event.issue.resolutionObject.name,
|
||||
comment)
|
||||
break
|
||||
case ISSUE_REOPENED_ID:
|
||||
content = String.format("%s **reopened** %s:\n\n> %s",
|
||||
author, issueUrlMd, comment)
|
||||
break
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
sendStreamMessage(zulipStream, subject, content)
|
||||
}
|
||||
|
||||
String post(String method, NameValuePair[] parameters) {
|
||||
PostMethod post = new PostMethod(zulipUrl(method))
|
||||
post.setRequestHeader("Content-Type", post.FORM_URL_ENCODED_CONTENT_TYPE)
|
||||
// TODO: Include more useful data in the User-agent
|
||||
post.setRequestHeader("User-agent", "ZulipJira/0.1")
|
||||
try {
|
||||
post.setRequestBody(parameters)
|
||||
HttpClient client = new HttpClient()
|
||||
client.executeMethod(post)
|
||||
String response = post.getResponseBodyAsString()
|
||||
if (post.getStatusCode() != HttpStatus.SC_OK) {
|
||||
String params = ""
|
||||
for (NameValuePair pair: parameters) {
|
||||
params += "\n" + pair.getName() + ":" + pair.getValue()
|
||||
}
|
||||
LOGGER.log(Level.SEVERE, "Error sending Zulip message:\n" + response + "\n\n" +
|
||||
"We sent:" + params)
|
||||
}
|
||||
return response;
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e)
|
||||
} finally {
|
||||
post.releaseConnection()
|
||||
}
|
||||
}
|
||||
|
||||
String truncate(String string, int length) {
|
||||
if (string.length() < length) {
|
||||
return string
|
||||
}
|
||||
return string.substring(0, length - 3) + "..."
|
||||
}
|
||||
|
||||
String sendStreamMessage(String stream, String subject, String message) {
|
||||
NameValuePair[] body = [new NameValuePair("api-key", zulipAPIKey),
|
||||
new NameValuePair("email", zulipEmail),
|
||||
new NameValuePair("type", "stream"),
|
||||
new NameValuePair("to", stream),
|
||||
new NameValuePair("subject", subject),
|
||||
new NameValuePair("content", message)]
|
||||
return post("send_message", body);
|
||||
}
|
||||
|
||||
String zulipUrl(method) {
|
||||
return base_url + "/v1/" + method
|
||||
}
|
||||
}
|
||||
49
api/integrations/nagios/nagios-notify-zulip
Executable file
49
api/integrations/nagios/nagios-notify-zulip
Executable file
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python
|
||||
import optparse
|
||||
import zulip
|
||||
|
||||
VERSION = "0.9"
|
||||
# Nagios passes the notification details as command line options.
|
||||
# In Nagios, "output" means "first line of output", and "long
|
||||
# output" means "other lines of output".
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option('--output', default='')
|
||||
parser.add_option('--long-output', default='')
|
||||
parser.add_option('--stream', default='nagios')
|
||||
parser.add_option('--config', default='/etc/nagios3/zuliprc')
|
||||
for opt in ('type', 'host', 'service', 'state'):
|
||||
parser.add_option('--' + opt)
|
||||
(opts, args) = parser.parse_args()
|
||||
|
||||
client = zulip.Client(config_file=opts.config, client="ZulipNagios/" + VERSION)
|
||||
|
||||
msg = dict(type='stream', to=opts.stream)
|
||||
|
||||
# Set a subject based on the host or service in question. This enables
|
||||
# threaded discussion of multiple concurrent issues, and provides useful
|
||||
# context when narrowed.
|
||||
#
|
||||
# We send PROBLEM and RECOVERY messages to the same subject.
|
||||
if opts.service is None:
|
||||
# Host notification
|
||||
thing = 'host'
|
||||
msg['subject'] = 'host %s' % (opts.host,)
|
||||
else:
|
||||
# Service notification
|
||||
thing = 'service'
|
||||
msg['subject'] = 'service %s on %s' % (opts.service, opts.host)
|
||||
|
||||
if len(msg['subject']) > 60:
|
||||
msg['subject'] = msg['subject'][0:57].rstrip() + "..."
|
||||
# e.g. **PROBLEM**: service is CRITICAL
|
||||
msg['content'] = '**%s**: %s is %s' % (opts.type, thing, opts.state)
|
||||
|
||||
# The "long output" can contain newlines represented by "\n" escape sequences.
|
||||
# The Nagios mail command uses /usr/bin/printf "%b" to expand these.
|
||||
# We will be more conservative and handle just this one escape sequence.
|
||||
output = (opts.output + '\n' + opts.long_output.replace(r'\n', '\n')).strip()
|
||||
if output:
|
||||
# Put any command output in a code block.
|
||||
msg['content'] += ('\n\n~~~~\n' + output + "\n~~~~\n")
|
||||
|
||||
client.send_message(msg)
|
||||
21
api/integrations/nagios/zulip_nagios.cfg
Normal file
21
api/integrations/nagios/zulip_nagios.cfg
Normal file
@@ -0,0 +1,21 @@
|
||||
define contact{
|
||||
contact_name zulip
|
||||
alias zulip
|
||||
service_notification_period 24x7
|
||||
host_notification_period 24x7
|
||||
service_notification_options w,u,c,r
|
||||
host_notification_options d,r
|
||||
service_notification_commands notify-service-by-zulip
|
||||
host_notification_commands notify-host-by-zulip
|
||||
}
|
||||
|
||||
# Zulip commands
|
||||
define command {
|
||||
command_name notify-host-by-zulip
|
||||
command_line /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --state="$HOSTSTATE$" --output="$HOSTOUTPUT$" --long-output="$LONGHOSTOUTPUT$"
|
||||
}
|
||||
|
||||
define command {
|
||||
command_name notify-service-by-zulip
|
||||
command_line /usr/local/share/zulip/integrations/nagios/nagios-notify-zulip --stream=nagios --type="$NOTIFICATIONTYPE$" --host="$HOSTADDRESS$" --service="$SERVICEDESC$" --state="$SERVICESTATE$" --output="$SERVICEOUTPUT$" --long-output="$LONGSERVICEOUTPUT$"
|
||||
}
|
||||
5
api/integrations/nagios/zuliprc.example
Normal file
5
api/integrations/nagios/zuliprc.example
Normal file
@@ -0,0 +1,5 @@
|
||||
# Fill these values in with the appropriate values for your realm, and
|
||||
# then install this value at /etc/nagios3/zuliprc
|
||||
[api]
|
||||
email = nagios-bot@example.com
|
||||
key = 0123456789abcdef0123456789abcdef
|
||||
3266
api/integrations/perforce/git_p4.py
Normal file
3266
api/integrations/perforce/git_p4.py
Normal file
File diff suppressed because it is too large
Load Diff
26
api/integrations/perforce/license.txt
Normal file
26
api/integrations/perforce/license.txt
Normal file
@@ -0,0 +1,26 @@
|
||||
git_p4.py was downloaded from https://raw.github.com/git/git/34022ba/git-p4.py
|
||||
|
||||
The header of that file references <http://opensource.org/licenses/mit-license.php>,
|
||||
the textual contents of which are included below.
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) <year> <copyright holders>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
88
api/integrations/perforce/zulip_change-commit.py
Executable file
88
api/integrations/perforce/zulip_change-commit.py
Executable file
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
'''Zulip notification change-commit hook.
|
||||
|
||||
In Perforce, The "change-commit" trigger is fired after a metadata has been
|
||||
created, files have been transferred, and the changelist comitted to the depot
|
||||
database.
|
||||
|
||||
This specific trigger expects command-line arguments in the form:
|
||||
%change% %changeroot%
|
||||
|
||||
For example:
|
||||
1234 //depot/security/src/
|
||||
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
import os.path
|
||||
|
||||
import git_p4
|
||||
|
||||
__version__ = "0.1"
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_perforce_config as config
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
|
||||
import zulip
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipPerforce/" + __version__)
|
||||
|
||||
try:
|
||||
changelist = int(sys.argv[1])
|
||||
changeroot = sys.argv[2]
|
||||
except IndexError:
|
||||
print >> sys.stderr, "Wrong number of arguments.\n\n",
|
||||
print >> sys.stderr, __doc__
|
||||
sys.exit(-1)
|
||||
except ValueError:
|
||||
print >> sys.stderr, "First argument must be an integer.\n\n",
|
||||
print >> sys.stderr, __doc__
|
||||
sys.exit(-1)
|
||||
|
||||
metadata = git_p4.p4_describe(changelist)
|
||||
|
||||
destination = config.commit_notice_destination(changeroot, changelist)
|
||||
if destination is None:
|
||||
# Don't forward the notice anywhere
|
||||
sys.exit(0)
|
||||
|
||||
message = """**{0}** committed revision @{1} to `{2}`.
|
||||
|
||||
> {3}
|
||||
""".format(metadata["user"], metadata["change"], changeroot, metadata["desc"])
|
||||
|
||||
message_data = {
|
||||
"type": "stream",
|
||||
"to": destination["stream"],
|
||||
"subject": destination["subject"],
|
||||
"content": message,
|
||||
}
|
||||
client.send_message(message_data)
|
||||
63
api/integrations/perforce/zulip_perforce_config.py
Normal file
63
api/integrations/perforce/zulip_perforce_config.py
Normal file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
# Change these values to configure authentication for the plugin
|
||||
ZULIP_USER = "p4-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# commit_notice_destination() lets you customize where commit notices
|
||||
# are sent to with the full power of a Python function.
|
||||
#
|
||||
# It takes the following arguments:
|
||||
# * path = the path to the Perforce depot on the server
|
||||
# * changelist = the changelist id
|
||||
#
|
||||
# Returns a dictionary encoding the stream and topic to send the
|
||||
# notification to (or None to send no notification).
|
||||
#
|
||||
# The default code below will send every commit except for ones in the
|
||||
# "master-plan" and "secret" subdirectories of //depot/ to:
|
||||
# * stream "depot_subdirectory-commits"
|
||||
# * subject "change_root"
|
||||
def commit_notice_destination(path, changelist):
|
||||
dirs = path.split('/')
|
||||
if len(dirs) >= 4 and dirs[3] not in ("*", "..."):
|
||||
directory = dirs[3]
|
||||
else:
|
||||
# No subdirectory, so just use "depot"
|
||||
directory = dirs[2]
|
||||
|
||||
if directory not in ["evil-master-plan", "my-super-secret-repository"]:
|
||||
return dict(stream = "%s-commits" % (directory,),
|
||||
subject = path)
|
||||
|
||||
# Return None for cases where you don't want a notice sent
|
||||
return None
|
||||
|
||||
## If properly installed, the Zulip API should be in your import
|
||||
## path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# This should not need to change unless you have a custom Zulip subdomain.
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
218
api/integrations/rss/rss-bot
Executable file
218
api/integrations/rss/rss-bot
Executable file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# RSS integration for Zulip
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import calendar
|
||||
import errno
|
||||
import hashlib
|
||||
from HTMLParser import HTMLParser
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urlparse
|
||||
|
||||
import feedparser
|
||||
import zulip
|
||||
VERSION = "0.9"
|
||||
RSS_DATA_DIR = os.path.expanduser(os.path.join('~', '.cache', 'zulip-rss'))
|
||||
OLDNESS_THRESHOLD = 30 # days
|
||||
|
||||
usage = """Usage: Send summaries of RSS entries for your favorite feeds to Zulip.
|
||||
|
||||
This bot requires the feedparser module.
|
||||
|
||||
To use this script:
|
||||
|
||||
1. Create an RSS feed file containing 1 feed URL per line (default feed
|
||||
file location: ~/.cache/zulip-rss/rss-feeds)
|
||||
2. Subscribe to the stream that will receive RSS updates (default stream: rss)
|
||||
3. create a ~/.zuliprc as described on https://zulip.com/api#api_keys
|
||||
4. Test the script by running it manually, like this:
|
||||
|
||||
/usr/local/share/zulip/integrations/rss/rss-bot
|
||||
|
||||
You can customize the location on the feed file and recipient stream, e.g.:
|
||||
|
||||
/usr/local/share/zulip/integrations/rss/rss-bot --feed-file=/path/to/my-feeds --stream=my-rss-stream
|
||||
|
||||
4. Configure a crontab entry for this script. A sample crontab entry for
|
||||
processing feeds stored in the default location and sending to the default
|
||||
stream every 5 minutes is:
|
||||
|
||||
*/5 * * * * /usr/local/share/zulip/integrations/rss/rss-bot"""
|
||||
|
||||
parser = optparse.OptionParser(usage)
|
||||
parser.add_option('--stream',
|
||||
dest='stream',
|
||||
help='The stream to which to send RSS messages.',
|
||||
default="rss",
|
||||
action='store')
|
||||
parser.add_option('--data-dir',
|
||||
dest='data_dir',
|
||||
help='The directory where feed metadata is stored',
|
||||
default=os.path.join(RSS_DATA_DIR),
|
||||
action='store')
|
||||
parser.add_option('--feed-file',
|
||||
dest='feed_file',
|
||||
help='The file containing a list of RSS feed URLs to follow, one URL per line',
|
||||
default=os.path.join(RSS_DATA_DIR, "rss-feeds"),
|
||||
action='store')
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(opts, args) = parser.parse_args()
|
||||
|
||||
def mkdir_p(path):
|
||||
# Python doesn't have an analog to `mkdir -p` < Python 3.2.
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError, e:
|
||||
if e.errno == errno.EEXIST and os.path.isdir(path):
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
try:
|
||||
mkdir_p(opts.data_dir)
|
||||
except OSError:
|
||||
# We can't write to the logfile, so just print and give up.
|
||||
print >>sys.stderr, "Unable to store RSS data at %s." % (opts.data_dir,)
|
||||
exit(1)
|
||||
|
||||
log_file = os.path.join(opts.data_dir, "rss-bot.log")
|
||||
log_format = "%(asctime)s: %(message)s"
|
||||
logging.basicConfig(format=log_format)
|
||||
|
||||
formatter = logging.Formatter(log_format)
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
def log_error_and_exit(error):
|
||||
logger.error(error)
|
||||
logger.error(usage)
|
||||
exit(1)
|
||||
|
||||
class MLStripper(HTMLParser):
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
self.fed = []
|
||||
|
||||
def handle_data(self, data):
|
||||
self.fed.append(data)
|
||||
|
||||
def get_data(self):
|
||||
return ''.join(self.fed)
|
||||
|
||||
def strip_tags(html):
|
||||
stripper = MLStripper()
|
||||
stripper.feed(html)
|
||||
return stripper.get_data()
|
||||
|
||||
def compute_entry_hash(entry):
|
||||
entry_time = entry.get("published", entry.get("updated"))
|
||||
entry_id = entry.get("id", entry.get("link"))
|
||||
return hashlib.md5(entry_id + entry_time).hexdigest()
|
||||
|
||||
def elide_subject(subject):
|
||||
MAX_TOPIC_LENGTH = 60
|
||||
if len(subject) > MAX_TOPIC_LENGTH:
|
||||
subject = subject[:MAX_TOPIC_LENGTH - 3].rstrip() + '...'
|
||||
return subject
|
||||
|
||||
def send_zulip(entry, feed_name):
|
||||
content = "**[%s](%s)**\n%s\n%s" % (entry.title,
|
||||
entry.link,
|
||||
strip_tags(entry.summary),
|
||||
entry.link)
|
||||
message = {"type": "stream",
|
||||
"sender": opts.email,
|
||||
"to": opts.stream,
|
||||
"subject": elide_subject(feed_name),
|
||||
"content": content,
|
||||
}
|
||||
return client.send_message(message)
|
||||
|
||||
try:
|
||||
with open(opts.feed_file, "r") as f:
|
||||
feed_urls = [feed.strip() for feed in f.readlines()]
|
||||
except IOError:
|
||||
log_error_and_exit("Unable to read feed file at %s." % (opts.feed_file,))
|
||||
|
||||
client = zulip.Client(email=opts.email, api_key=opts.api_key,
|
||||
site=opts.site, client="ZulipRSS/" + VERSION)
|
||||
|
||||
first_message = True
|
||||
|
||||
for feed_url in feed_urls:
|
||||
feed_file = os.path.join(opts.data_dir, urlparse.urlparse(feed_url).netloc)
|
||||
|
||||
try:
|
||||
with open(feed_file, "r") as f:
|
||||
old_feed_hashes = dict((line.strip(), True) for line in f.readlines())
|
||||
except IOError:
|
||||
old_feed_hashes = {}
|
||||
|
||||
new_hashes = []
|
||||
data = feedparser.parse(feed_url)
|
||||
|
||||
for entry in data.entries:
|
||||
entry_hash = compute_entry_hash(entry)
|
||||
# An entry has either been published or updated.
|
||||
entry_time = entry.get("published_parsed", entry.get("updated_parsed"))
|
||||
if (time.time() - calendar.timegm(entry_time)) > OLDNESS_THRESHOLD * 60 * 60 * 24:
|
||||
# As a safeguard against misbehaving feeds, don't try to process
|
||||
# entries older than some threshold.
|
||||
continue
|
||||
if entry_hash in old_feed_hashes:
|
||||
# We've already seen this. No need to process any older entries.
|
||||
break
|
||||
if (not old_feed_hashes) and (len(new_hashes) >= 3):
|
||||
# On a first run, pick up the 3 most recent entries. An RSS feed has
|
||||
# entries in reverse chronological order.
|
||||
break
|
||||
|
||||
feed_name = data.feed.title or feed_url
|
||||
|
||||
response = send_zulip(entry, feed_name)
|
||||
if response["result"] != "success":
|
||||
logger.error("Error processing %s" % (feed_url,))
|
||||
logger.error(response)
|
||||
if first_message:
|
||||
# This is probably some fundamental problem like the stream not
|
||||
# existing or something being misconfigured, so bail instead of
|
||||
# getting the same error for every RSS entry.
|
||||
log_error_and_exit("Failed to process first message")
|
||||
# Go ahead and move on -- perhaps this entry is corrupt.
|
||||
new_hashes.append(entry_hash)
|
||||
first_message = False
|
||||
|
||||
with open(feed_file, "a") as f:
|
||||
for hash in new_hashes:
|
||||
f.write(hash + "\n")
|
||||
|
||||
logger.info("Sent zulips for %d %s entries" % (len(new_hashes), feed_url))
|
||||
71
api/integrations/svn/post-commit
Executable file
71
api/integrations/svn/post-commit
Executable file
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Zulip notification post-commit hook.
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
# The "post-commit" script is run after a transaction is completed and a new
|
||||
# revision is created. It is passed arguments on the command line in this
|
||||
# form:
|
||||
# <path> <revision>
|
||||
# For example:
|
||||
# /srv/svn/carols 1843
|
||||
|
||||
import os
|
||||
import sys
|
||||
import os.path
|
||||
import pysvn
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_svn_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
|
||||
import zulip
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipSVN/" + VERSION)
|
||||
svn = pysvn.Client()
|
||||
|
||||
path, rev = sys.argv[1:]
|
||||
|
||||
# since its a local path, prepend "file://"
|
||||
path = "file://" + path
|
||||
|
||||
entry = svn.log(path, revision_end=pysvn.Revision(pysvn.opt_revision_kind.number, rev))[0]
|
||||
message = """**{0}** committed revision r{1} to `{2}`.
|
||||
|
||||
> {3}
|
||||
""".format(entry['author'], rev, path.split('/')[-1], entry['revprops']['svn:log'])
|
||||
|
||||
destination = config.commit_notice_destination(path, rev)
|
||||
|
||||
message_data = {
|
||||
"type": "stream",
|
||||
"to": destination["stream"],
|
||||
"subject": destination["subject"],
|
||||
"content": message,
|
||||
}
|
||||
client.send_message(message_data)
|
||||
57
api/integrations/svn/zulip_svn_config.py
Normal file
57
api/integrations/svn/zulip_svn_config.py
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
# Change these values to configure authentication for the plugin
|
||||
ZULIP_USER = "svn-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
# commit_notice_destination() lets you customize where commit notices
|
||||
# are sent to with the full power of a Python function.
|
||||
#
|
||||
# It takes the following arguments:
|
||||
# * path = the path to the svn repository on the server
|
||||
# * commit = the commit id
|
||||
#
|
||||
# Returns a dictionary encoding the stream and subject to send the
|
||||
# notification to (or None to send no notification).
|
||||
#
|
||||
# The default code below will send every commit except for the "evil-master-plan"
|
||||
# and "my-super-secret-repository" repos to
|
||||
# * stream "commits"
|
||||
# * topic "branch_name"
|
||||
def commit_notice_destination(path, commit):
|
||||
repo = path.split('/')[-1]
|
||||
if repo not in ["evil-master-plan", "my-super-secret-repository"]:
|
||||
return dict(stream = "commits",
|
||||
subject = u"%s" % (repo,))
|
||||
|
||||
# Return None for cases where you don't want a notice sent
|
||||
return None
|
||||
|
||||
## If properly installed, the Zulip API should be in your import
|
||||
## path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
126
api/integrations/trac/zulip_trac.py
Normal file
126
api/integrations/trac/zulip_trac.py
Normal file
@@ -0,0 +1,126 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
|
||||
# Zulip trac plugin -- sends zulips when tickets change.
|
||||
#
|
||||
# Install by copying this file and zulip_trac_config.py to the trac
|
||||
# plugins/ subdirectory, customizing the constants in
|
||||
# zulip_trac_config.py, and then adding "zulip_trac" to the
|
||||
# components section of the conf/trac.ini file, like so:
|
||||
#
|
||||
# [components]
|
||||
# zulip_trac = enabled
|
||||
#
|
||||
# You may then need to restart trac (or restart Apache) for the bot
|
||||
# (or changes to the bot) to actually be loaded by trac.
|
||||
|
||||
from trac.core import Component, implements
|
||||
from trac.ticket import ITicketChangeListener
|
||||
import sys
|
||||
import os.path
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
import zulip_trac_config as config
|
||||
VERSION = "0.9"
|
||||
|
||||
if config.ZULIP_API_PATH is not None:
|
||||
sys.path.append(config.ZULIP_API_PATH)
|
||||
|
||||
import zulip
|
||||
client = zulip.Client(
|
||||
email=config.ZULIP_USER,
|
||||
site=config.ZULIP_SITE,
|
||||
api_key=config.ZULIP_API_KEY,
|
||||
client="ZulipTrac/" + VERSION)
|
||||
|
||||
def markdown_ticket_url(ticket, heading="ticket"):
|
||||
return "[%s #%s](%s/%s)" % (heading, ticket.id, config.TRAC_BASE_TICKET_URL, ticket.id)
|
||||
|
||||
def markdown_block(desc):
|
||||
return "\n\n>" + "\n> ".join(desc.split("\n")) + "\n"
|
||||
|
||||
def truncate(string, length):
|
||||
if len(string) <= length:
|
||||
return string
|
||||
return string[:length - 3] + "..."
|
||||
|
||||
def trac_subject(ticket):
|
||||
return truncate("#%s: %s" % (ticket.id, ticket.values.get("summary")), 60)
|
||||
|
||||
def send_update(ticket, content):
|
||||
client.send_message({
|
||||
"type": "stream",
|
||||
"to": config.STREAM_FOR_NOTIFICATIONS,
|
||||
"content": content,
|
||||
"subject": trac_subject(ticket)
|
||||
})
|
||||
|
||||
class ZulipPlugin(Component):
|
||||
implements(ITicketChangeListener)
|
||||
|
||||
def ticket_created(self, ticket):
|
||||
"""Called when a ticket is created."""
|
||||
content = "%s created %s in component **%s**, priority **%s**:\n" % \
|
||||
(ticket.values.get("reporter"), markdown_ticket_url(ticket),
|
||||
ticket.values.get("component"), ticket.values.get("priority"))
|
||||
# Include the full subject if it will be truncated
|
||||
if len(ticket.values.get("summary")) > 60:
|
||||
content += "**%s**\n" % (ticket.values.get("summary"),)
|
||||
if ticket.values.get("description") != "":
|
||||
content += "%s" % (markdown_block(ticket.values.get("description")),)
|
||||
send_update(ticket, content)
|
||||
|
||||
def ticket_changed(self, ticket, comment, author, old_values):
|
||||
"""Called when a ticket is modified.
|
||||
|
||||
`old_values` is a dictionary containing the previous values of the
|
||||
fields that have changed.
|
||||
"""
|
||||
if not (set(old_values.keys()).intersection(set(config.TRAC_NOTIFY_FIELDS)) or
|
||||
(comment and "comment" in set(config.TRAC_NOTIFY_FIELDS))):
|
||||
return
|
||||
|
||||
content = "%s updated %s" % (author, markdown_ticket_url(ticket))
|
||||
if comment:
|
||||
content += ' with comment: %s\n\n' % (markdown_block(comment),)
|
||||
else:
|
||||
content += ":\n\n"
|
||||
field_changes = []
|
||||
for key in old_values.keys():
|
||||
if key == "description":
|
||||
content += '- Changed %s from %s to %s' % (key, markdown_block(old_values.get(key)),
|
||||
markdown_block(ticket.values.get(key)))
|
||||
elif old_values.get(key) == "":
|
||||
field_changes.append('%s: => **%s**' % (key, ticket.values.get(key)))
|
||||
elif ticket.values.get(key) == "":
|
||||
field_changes.append('%s: **%s** => ""' % (key, old_values.get(key)))
|
||||
else:
|
||||
field_changes.append('%s: **%s** => **%s**' % (key, old_values.get(key),
|
||||
ticket.values.get(key)))
|
||||
content += ", ".join(field_changes)
|
||||
|
||||
send_update(ticket, content)
|
||||
|
||||
def ticket_deleted(self, ticket):
|
||||
"""Called when a ticket is deleted."""
|
||||
content = "%s was deleted." % markdown_ticket_url(ticket, heading="Ticket")
|
||||
send_update(ticket, content)
|
||||
51
api/integrations/trac/zulip_trac_config.py
Normal file
51
api/integrations/trac/zulip_trac_config.py
Normal file
@@ -0,0 +1,51 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Copyright © 2012 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
# See zulip_trac.py for installation and configuration instructions
|
||||
|
||||
# Change these constants to configure the plugin:
|
||||
ZULIP_USER = "trac-bot@example.com"
|
||||
ZULIP_API_KEY = "0123456789abcdef0123456789abcdef"
|
||||
STREAM_FOR_NOTIFICATIONS = "trac"
|
||||
TRAC_BASE_TICKET_URL = "https://trac.example.com/ticket"
|
||||
|
||||
# Most people find that having every change in Trac result in a
|
||||
# notification is too noisy -- in particular, when someone goes
|
||||
# through recategorizing a bunch of tickets, that can often be noisy
|
||||
# and annoying. We solve this issue by only sending a notification
|
||||
# for changes to the fields listed below.
|
||||
#
|
||||
# Total list of possible fields is:
|
||||
# (priority, milestone, cc, owner, keywords, component, severity,
|
||||
# type, versions, description, resolution, summary, comment)
|
||||
#
|
||||
# The following is the list of fields which can be changed without
|
||||
# triggering a Zulip notification; change these to match your team's
|
||||
# workflow.
|
||||
TRAC_NOTIFY_FIELDS = ["description", "summary", "resolution", "comment", "owner"]
|
||||
|
||||
## If properly installed, the Zulip API should be in your import
|
||||
## path, but if not, set a custom path below
|
||||
ZULIP_API_PATH = None
|
||||
|
||||
# If you're using Zulip Enterprise, set this to your Zulip Enterprise server
|
||||
ZULIP_SITE = "https://api.zulip.com"
|
||||
162
api/integrations/twitter/twitter-bot
Executable file
162
api/integrations/twitter/twitter-bot
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Twitter integration for Zulip
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import optparse
|
||||
import ConfigParser
|
||||
|
||||
import zulip
|
||||
VERSION = "0.9"
|
||||
CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
|
||||
|
||||
def write_config(config, since_id, user):
|
||||
config.set('twitter', 'since_id', since_id)
|
||||
config.set('twitter', 'user_id', user)
|
||||
with open(CONFIGFILE, 'wb') as configfile:
|
||||
config.write(configfile)
|
||||
|
||||
parser = optparse.OptionParser(r"""
|
||||
|
||||
%prog --user foo@example.com --api-key 0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --twitter-id twitter_handle
|
||||
|
||||
Slurp tweets on your timeline into a specific zulip stream.
|
||||
|
||||
Run this on your personal machine. Your API key and twitter id are revealed to local
|
||||
users through the command line or config file.
|
||||
|
||||
This bot uses OAuth to authenticate with twitter. Please create a ~/.zulip_twitterrc with
|
||||
the following contents:
|
||||
|
||||
[twitter]
|
||||
consumer_key =
|
||||
consumer_secret =
|
||||
access_token_key =
|
||||
access_token_secret =
|
||||
|
||||
In order to obtain a consumer key & secret, you must register a new application under your twitter account:
|
||||
|
||||
1. Go to http://dev.twitter.com
|
||||
2. Log in
|
||||
3. In the menu under your username, click My Applications
|
||||
4. Create a new application
|
||||
|
||||
Make sure to go the application you created and click "create my access token" as well. Fill in the values displayed.
|
||||
|
||||
Depends on: twitter-python
|
||||
""")
|
||||
|
||||
parser.add_option('--twitter-id',
|
||||
help='Twitter username to poll for new tweets from"',
|
||||
metavar='URL')
|
||||
parser.add_option('--stream',
|
||||
help='Default zulip stream to write tweets to')
|
||||
parser.add_option('--limit-tweets',
|
||||
default=15,
|
||||
type='int',
|
||||
help='Maximum number of tweets to push at once')
|
||||
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if not options.twitter_id:
|
||||
parser.error('You must specify --twitter-id')
|
||||
|
||||
try:
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.read(CONFIGFILE)
|
||||
|
||||
consumer_key = config.get('twitter', 'consumer_key')
|
||||
consumer_secret = config.get('twitter', 'consumer_secret')
|
||||
access_token_key = config.get('twitter', 'access_token_key')
|
||||
access_token_secret = config.get('twitter', 'access_token_secret')
|
||||
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
|
||||
parser.error("Please provide a ~/.zulip_twitterrc")
|
||||
|
||||
if not consumer_key or not consumer_secret or not access_token_key or not access_token_secret:
|
||||
parser.error("Please provide a ~/.zulip_twitterrc")
|
||||
|
||||
try:
|
||||
import twitter
|
||||
except ImportError:
|
||||
parser.error("Please install twitter-python")
|
||||
|
||||
api = twitter.Api(consumer_key=consumer_key,
|
||||
consumer_secret=consumer_secret,
|
||||
access_token_key=access_token_key,
|
||||
access_token_secret=access_token_secret)
|
||||
|
||||
|
||||
user = api.VerifyCredentials()
|
||||
|
||||
if not user.GetId():
|
||||
print "Unable to log in to twitter with supplied credentials. Please double-check and try again"
|
||||
sys.exit()
|
||||
|
||||
try:
|
||||
since_id = config.getint('twitter', 'since_id')
|
||||
except ConfigParser.NoOptionError:
|
||||
since_id = -1
|
||||
|
||||
try:
|
||||
user_id = config.get('twitter', 'user_id')
|
||||
except ConfigParser.NoOptionError:
|
||||
user_id = options.twitter_id
|
||||
|
||||
client = zulip.Client(
|
||||
email=options.email,
|
||||
api_key=options.api_key,
|
||||
site=options.site,
|
||||
client="ZulipTwitter/" + VERSION,
|
||||
verbose=True)
|
||||
|
||||
if since_id < 0 or options.twitter_id != user_id:
|
||||
# No since id yet, fetch the latest and then start monitoring from next time
|
||||
# Or, a different user id is being asked for, so start from scratch
|
||||
# Either way, fetch last 5 tweets to start off
|
||||
statuses = api.GetFriendsTimeline(user=options.twitter_id, count=5)
|
||||
else:
|
||||
# We have a saved last id, so insert all newer tweets into the zulip stream
|
||||
statuses = api.GetFriendsTimeline(user=options.twitter_id, since_id=since_id)
|
||||
|
||||
for status in statuses[::-1][:options.limit_tweets]:
|
||||
composed = "%s (%s)" % (status.GetUser().GetName(), status.GetUser().GetScreenName())
|
||||
message = {
|
||||
"type": "stream",
|
||||
"to": [options.stream],
|
||||
"subject": composed,
|
||||
"content": status.GetText(),
|
||||
}
|
||||
|
||||
ret = client.send_message(message)
|
||||
|
||||
if ret['result'] == 'error':
|
||||
# If sending failed (e.g. no such stream), abort and retry next time
|
||||
print "Error sending message to zulip: %s" % ret['msg']
|
||||
break
|
||||
else:
|
||||
since_id = status.GetId()
|
||||
|
||||
write_config(config, since_id, user_id)
|
||||
190
api/integrations/twitter/twitter-search-bot
Executable file
190
api/integrations/twitter/twitter-search-bot
Executable file
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Twitter search integration for Zulip
|
||||
#
|
||||
# Copyright © 2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import optparse
|
||||
import ConfigParser
|
||||
|
||||
import zulip
|
||||
VERSION = "0.9"
|
||||
CONFIGFILE = os.path.expanduser("~/.zulip_twitterrc")
|
||||
|
||||
def write_config(config, since_id):
|
||||
if 'search' not in config.sections():
|
||||
config.add_section('search')
|
||||
config.set('search', 'since_id', since_id)
|
||||
with open(CONFIGFILE, 'wb') as configfile:
|
||||
config.write(configfile)
|
||||
|
||||
parser = optparse.OptionParser(r"""
|
||||
|
||||
%prog --user foo@zulip.com --api-key a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5 --search="@nprnews,quantum physics"
|
||||
|
||||
Send Twitter search results to a Zulip stream.
|
||||
|
||||
Depends on: twitter-python version 1.0 or later
|
||||
|
||||
To use this script:
|
||||
|
||||
1. Set up Twitter authentication, as described below
|
||||
2. Subscribe to the stream that will receive Twitter updates (default stream: twitter)
|
||||
3. Test the script by running it manually, like this:
|
||||
|
||||
/usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
|
||||
|
||||
4. Configure a crontab entry for this script. A sample crontab entry
|
||||
that will process tweets every 5 minutes is:
|
||||
|
||||
*/5 * * * * /usr/local/share/zulip/integrations/twitter/twitter-search-bot --search="@nprnews,quantum physics"
|
||||
|
||||
== Setting up Twitter authentications ==
|
||||
|
||||
Run this on a personal or trusted machine, because your API key is
|
||||
visible to local users through the command line or config file.
|
||||
|
||||
This bot uses OAuth to authenticate with Twitter. Please create a
|
||||
~/.zulip_twitterrc with the following contents:
|
||||
|
||||
[twitter]
|
||||
consumer_key =
|
||||
consumer_secret =
|
||||
access_token_key =
|
||||
access_token_secret =
|
||||
|
||||
In order to obtain a consumer key & secret, you must register a
|
||||
new application under your Twitter account:
|
||||
|
||||
1. Go to http://dev.twitter.com
|
||||
2. Log in
|
||||
3. In the menu under your username, click My Applications
|
||||
4. Create a new application
|
||||
|
||||
Make sure to go the application you created and click "create my
|
||||
access token" as well. Fill in the values displayed.
|
||||
""")
|
||||
|
||||
parser.add_option('--search',
|
||||
dest='search_terms',
|
||||
help='Terms to search on',
|
||||
action='store')
|
||||
parser.add_option('--stream',
|
||||
dest='stream',
|
||||
help='The stream to which to send tweets',
|
||||
default="twitter",
|
||||
action='store')
|
||||
parser.add_option('--limit-tweets',
|
||||
default=15,
|
||||
type='int',
|
||||
help='Maximum number of tweets to send at once')
|
||||
|
||||
parser.add_option_group(zulip.generate_option_group(parser))
|
||||
(opts, args) = parser.parse_args()
|
||||
|
||||
if not opts.search_terms:
|
||||
parser.error('You must specify a search term.')
|
||||
|
||||
try:
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.read(CONFIGFILE)
|
||||
|
||||
consumer_key = config.get('twitter', 'consumer_key')
|
||||
consumer_secret = config.get('twitter', 'consumer_secret')
|
||||
access_token_key = config.get('twitter', 'access_token_key')
|
||||
access_token_secret = config.get('twitter', 'access_token_secret')
|
||||
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
|
||||
parser.error("Please provide a ~/.zulip_twitterrc")
|
||||
|
||||
if not (consumer_key and consumer_secret and access_token_key and access_token_secret):
|
||||
parser.error("Please provide a ~/.zulip_twitterrc")
|
||||
|
||||
try:
|
||||
since_id = config.getint('search', 'since_id')
|
||||
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
|
||||
since_id = 0
|
||||
|
||||
try:
|
||||
import twitter
|
||||
except ImportError:
|
||||
parser.error("Please install twitter-python")
|
||||
|
||||
api = twitter.Api(consumer_key=consumer_key,
|
||||
consumer_secret=consumer_secret,
|
||||
access_token_key=access_token_key,
|
||||
access_token_secret=access_token_secret)
|
||||
|
||||
user = api.VerifyCredentials()
|
||||
|
||||
if not user.GetId():
|
||||
print "Unable to log in to Twitter with supplied credentials.\
|
||||
Please double-check and try again."
|
||||
sys.exit()
|
||||
|
||||
client = zulip.Client(
|
||||
email=opts.email,
|
||||
api_key=opts.api_key,
|
||||
site=opts.site,
|
||||
client="ZulipTwitterSearch/" + VERSION,
|
||||
verbose=True)
|
||||
|
||||
search_query = " OR ".join(opts.search_terms.split(","))
|
||||
statuses = api.GetSearch(search_query, since_id=since_id)
|
||||
|
||||
for status in statuses[::-1][:opts.limit_tweets]:
|
||||
# https://twitter.com/eatevilpenguins/status/309995853408530432
|
||||
composed = "%s (%s)" % (status.GetUser().GetName(),
|
||||
status.GetUser().GetScreenName())
|
||||
url = "https://twitter.com/%s/status/%s" % (status.GetUser().GetScreenName(),
|
||||
status.GetId())
|
||||
content = status.GetText()
|
||||
|
||||
search_term_used = None
|
||||
for term in opts.search_terms.split(","):
|
||||
if term.lower() in content.lower():
|
||||
search_term_used = term
|
||||
break
|
||||
# For some reason (perhaps encodings or message tranformations we
|
||||
# didn't anticipate), we don't know what term was used, so use a
|
||||
# default.
|
||||
if not search_term_used:
|
||||
search_term_used = "mentions"
|
||||
|
||||
message = {
|
||||
"type": "stream",
|
||||
"to": [opts.stream],
|
||||
"subject": search_term_used,
|
||||
"content": url,
|
||||
}
|
||||
|
||||
ret = client.send_message(message)
|
||||
|
||||
if ret['result'] == 'error':
|
||||
# If sending failed (e.g. no such stream), abort and retry next time
|
||||
print "Error sending message to zulip: %s" % ret['msg']
|
||||
break
|
||||
else:
|
||||
since_id = status.GetId()
|
||||
|
||||
write_config(config, since_id)
|
||||
3
api/setup.cfg
Normal file
3
api/setup.cfg
Normal file
@@ -0,0 +1,3 @@
|
||||
# This way our scripts are installed in the data directory
|
||||
[aliases]
|
||||
install = install_data install
|
||||
73
api/setup.py
Normal file
73
api/setup.py
Normal file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import itertools
|
||||
|
||||
def version():
|
||||
version_py = os.path.join(os.path.dirname(__file__), "zulip", "__init__.py")
|
||||
with open(version_py) as in_handle:
|
||||
version_line = itertools.dropwhile(lambda x: not x.startswith("__version__"),
|
||||
in_handle).next()
|
||||
version = version_line.split('=')[-1].strip().replace('"', '')
|
||||
return version
|
||||
|
||||
def recur_expand(target_root, dir):
|
||||
for root, _, files in os.walk(dir):
|
||||
paths = [os.path.join(root, f) for f in files]
|
||||
if len(paths):
|
||||
yield os.path.join(target_root, root), paths
|
||||
|
||||
# We should be installable with either setuptools or distutils.
|
||||
package_info = dict(
|
||||
name='zulip',
|
||||
version=version(),
|
||||
description='Bindings for the Zulip message API',
|
||||
author='Zulip, Inc.',
|
||||
author_email='support@zulip.com',
|
||||
classifiers=[
|
||||
'Development Status :: 3 - Alpha',
|
||||
'Environment :: Web Environment',
|
||||
'Intended Audience :: Developers',
|
||||
'License :: OSI Approved :: MIT License',
|
||||
'Topic :: Communications :: Chat',
|
||||
],
|
||||
url='https://www.zulip.com/dist/api/',
|
||||
packages=['zulip'],
|
||||
data_files=[('share/zulip/examples', ["examples/zuliprc", "examples/send-message", "examples/subscribe",
|
||||
"examples/get-public-streams", "examples/unsubscribe",
|
||||
"examples/list-members", "examples/list-subscriptions",
|
||||
"examples/print-messages"])] + \
|
||||
list(recur_expand('share/zulip', 'integrations/')),
|
||||
scripts=["bin/zulip-send"],
|
||||
)
|
||||
|
||||
setuptools_info = dict(
|
||||
install_requires=['requests>=0.12.1',
|
||||
'simplejson',
|
||||
],
|
||||
)
|
||||
|
||||
try:
|
||||
from setuptools import setup
|
||||
package_info.update(setuptools_info)
|
||||
except ImportError:
|
||||
from distutils.core import setup
|
||||
from distutils.version import LooseVersion
|
||||
# Manual dependency check
|
||||
try:
|
||||
import simplejson
|
||||
except ImportError:
|
||||
print >>sys.stderr, "simplejson is not installed"
|
||||
sys.exit(1)
|
||||
try:
|
||||
import requests
|
||||
assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
|
||||
except (ImportError, AssertionError):
|
||||
print >>sys.stderr, "requests >=0.12.1 is not installed"
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
setup(**package_info)
|
||||
363
api/zulip/__init__.py
Normal file
363
api/zulip/__init__.py
Normal file
@@ -0,0 +1,363 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright © 2012-2014 Zulip, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
import simplejson
|
||||
import requests
|
||||
import time
|
||||
import traceback
|
||||
import urlparse
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import platform
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from ConfigParser import SafeConfigParser
|
||||
import logging
|
||||
|
||||
|
||||
__version__ = "0.2.4"
|
||||
|
||||
# Check that we have a recent enough version
|
||||
# Older versions don't provide the 'json' attribute on responses.
|
||||
assert(LooseVersion(requests.__version__) >= LooseVersion('0.12.1'))
|
||||
# In newer versions, the 'json' attribute is a function, not a property
|
||||
requests_json_is_function = callable(requests.Response.json)
|
||||
|
||||
API_VERSTRING = "v1/"
|
||||
|
||||
def _default_client():
|
||||
return "ZulipPython/" + __version__
|
||||
|
||||
def generate_option_group(parser):
|
||||
group = optparse.OptionGroup(parser, 'API configuration')
|
||||
group.add_option('--site',
|
||||
help="Zulip Enterprise server URI (if using Zulip Enterprise)",
|
||||
default=None)
|
||||
group.add_option('--api-key',
|
||||
action='store')
|
||||
group.add_option('--user',
|
||||
dest='email',
|
||||
help='Email address of the calling bot or user.')
|
||||
group.add_option('--config-file',
|
||||
action='store',
|
||||
help='Location of an ini file containing the\nabove information. (default ~/.zuliprc)')
|
||||
group.add_option('-v', '--verbose',
|
||||
action='store_true',
|
||||
help='Provide detailed output.')
|
||||
group.add_option('--client',
|
||||
action='store',
|
||||
default=None,
|
||||
help=optparse.SUPPRESS_HELP)
|
||||
return group
|
||||
|
||||
def init_from_options(options, client=None):
|
||||
if options.client is not None:
|
||||
client = options.client
|
||||
elif client is None:
|
||||
client = _default_client()
|
||||
return Client(email=options.email, api_key=options.api_key,
|
||||
config_file=options.config_file, verbose=options.verbose,
|
||||
site=options.site, client=client)
|
||||
|
||||
class Client(object):
|
||||
def __init__(self, email=None, api_key=None, config_file=None,
|
||||
verbose=False, retry_on_errors=True,
|
||||
site=None, client=None):
|
||||
if client is None:
|
||||
client = _default_client()
|
||||
if None in (api_key, email):
|
||||
if config_file is None:
|
||||
config_file = os.path.join(os.environ["HOME"], ".zuliprc")
|
||||
if (not os.path.exists(config_file) and
|
||||
os.path.exists(os.path.join(os.environ["HOME"], ".humbugrc"))):
|
||||
raise RuntimeError("The Zulip API configuration file is now ~/.zuliprc; please run:\n\n mv ~/.humbugrc ~/.zuliprc\n")
|
||||
if not os.path.exists(config_file):
|
||||
raise RuntimeError("api_key or email not specified and %s does not exist"
|
||||
% (config_file,))
|
||||
config = SafeConfigParser()
|
||||
with file(config_file, 'r') as f:
|
||||
config.readfp(f, config_file)
|
||||
if api_key is None:
|
||||
api_key = config.get("api", "key")
|
||||
if email is None:
|
||||
email = config.get("api", "email")
|
||||
if site is None and config.has_option("api", "site"):
|
||||
site = config.get("api", "site")
|
||||
|
||||
self.api_key = api_key
|
||||
self.email = email
|
||||
self.verbose = verbose
|
||||
if site is not None:
|
||||
if not site.startswith("http"):
|
||||
site = "https://" + site
|
||||
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
|
||||
site = site.rstrip("/")
|
||||
self.base_url = site
|
||||
else:
|
||||
self.base_url = "https://api.zulip.com"
|
||||
|
||||
if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
|
||||
self.base_url += "/api"
|
||||
self.base_url += "/"
|
||||
self.retry_on_errors = retry_on_errors
|
||||
self.client_name = client
|
||||
|
||||
def get_user_agent(self):
|
||||
vendor = platform.system()
|
||||
vendor_version = platform.release()
|
||||
|
||||
if vendor == "Linux":
|
||||
vendor, vendor_version, dummy = platform.linux_distribution()
|
||||
elif vendor == "Windows":
|
||||
vendor_version = platform.win32_ver()[1]
|
||||
elif vendor == "Darwin":
|
||||
vendor_version = platform.mac_ver()[0]
|
||||
|
||||
return "{client_name} ({vendor}; {vendor_version})".format(
|
||||
client_name=self.client_name,
|
||||
vendor=vendor,
|
||||
vendor_version=vendor_version,
|
||||
)
|
||||
|
||||
def do_api_query(self, orig_request, url, method="POST", longpolling = False):
|
||||
request = {}
|
||||
|
||||
for (key, val) in orig_request.iteritems():
|
||||
if not (isinstance(val, str) or isinstance(val, unicode)):
|
||||
request[key] = simplejson.dumps(val)
|
||||
else:
|
||||
request[key] = val
|
||||
|
||||
query_state = {
|
||||
'had_error_retry': False,
|
||||
'request': request,
|
||||
'failures': 0,
|
||||
}
|
||||
|
||||
def error_retry(error_string):
|
||||
if not self.retry_on_errors or query_state["failures"] >= 10:
|
||||
return False
|
||||
if self.verbose:
|
||||
if not query_state["had_error_retry"]:
|
||||
sys.stdout.write("zulip API(%s): connection error%s -- retrying." % \
|
||||
(url.split(API_VERSTRING, 2)[0], error_string,))
|
||||
query_state["had_error_retry"] = True
|
||||
else:
|
||||
sys.stdout.write(".")
|
||||
sys.stdout.flush()
|
||||
query_state["request"]["dont_block"] = simplejson.dumps(True)
|
||||
time.sleep(1)
|
||||
query_state["failures"] += 1
|
||||
return True
|
||||
|
||||
def end_error_retry(succeeded):
|
||||
if query_state["had_error_retry"] and self.verbose:
|
||||
if succeeded:
|
||||
print "Success!"
|
||||
else:
|
||||
print "Failed!"
|
||||
|
||||
while True:
|
||||
try:
|
||||
if method == "GET":
|
||||
kwarg = "params"
|
||||
else:
|
||||
kwarg = "data"
|
||||
kwargs = {kwarg: query_state["request"]}
|
||||
res = requests.request(
|
||||
method,
|
||||
urlparse.urljoin(self.base_url, url),
|
||||
auth=requests.auth.HTTPBasicAuth(self.email,
|
||||
self.api_key),
|
||||
verify=True, timeout=90,
|
||||
headers={"User-agent": self.get_user_agent()},
|
||||
**kwargs)
|
||||
|
||||
# On 50x errors, try again after a short sleep
|
||||
if str(res.status_code).startswith('5'):
|
||||
if error_retry(" (server %s)" % (res.status_code,)):
|
||||
continue
|
||||
# Otherwise fall through and process the python-requests error normally
|
||||
except (requests.exceptions.Timeout, requests.exceptions.SSLError) as e:
|
||||
# Timeouts are either a Timeout or an SSLError; we
|
||||
# want the later exception handlers to deal with any
|
||||
# non-timeout other SSLErrors
|
||||
if (isinstance(e, requests.exceptions.SSLError) and
|
||||
str(e) != "The read operation timed out"):
|
||||
raise
|
||||
if longpolling:
|
||||
# When longpolling, we expect the timeout to fire,
|
||||
# and the correct response is to just retry
|
||||
continue
|
||||
else:
|
||||
end_error_retry(False)
|
||||
return {'msg': "Connection error:\n%s" % traceback.format_exc(),
|
||||
"result": "connection-error"}
|
||||
except requests.exceptions.ConnectionError:
|
||||
if error_retry(""):
|
||||
continue
|
||||
end_error_retry(False)
|
||||
return {'msg': "Connection error:\n%s" % traceback.format_exc(),
|
||||
"result": "connection-error"}
|
||||
except Exception:
|
||||
# We'll split this out into more cases as we encounter new bugs.
|
||||
return {'msg': "Unexpected error:\n%s" % traceback.format_exc(),
|
||||
"result": "unexpected-error"}
|
||||
|
||||
try:
|
||||
if requests_json_is_function:
|
||||
json_result = res.json()
|
||||
else:
|
||||
json_result = res.json
|
||||
except Exception:
|
||||
json_result = None
|
||||
|
||||
if json_result is not None:
|
||||
end_error_retry(True)
|
||||
return json_result
|
||||
end_error_retry(False)
|
||||
return {'msg': "Unexpected error from the server", "result": "http-error",
|
||||
"status_code": res.status_code}
|
||||
|
||||
@classmethod
|
||||
def _register(cls, name, url=None, make_request=(lambda request={}: request),
|
||||
method="POST", **query_kwargs):
|
||||
if url is None:
|
||||
url = name
|
||||
def call(self, *args, **kwargs):
|
||||
request = make_request(*args, **kwargs)
|
||||
return self.do_api_query(request, API_VERSTRING + url, method=method, **query_kwargs)
|
||||
call.func_name = name
|
||||
setattr(cls, name, call)
|
||||
|
||||
def call_on_each_event(self, callback, event_types=None, narrow=[]):
|
||||
def do_register():
|
||||
while True:
|
||||
if event_types is None:
|
||||
res = self.register()
|
||||
else:
|
||||
res = self.register(event_types=event_types, narrow=narrow)
|
||||
|
||||
if 'error' in res.get('result'):
|
||||
if self.verbose:
|
||||
print "Server returned error:\n%s" % res['msg']
|
||||
time.sleep(1)
|
||||
else:
|
||||
return (res['queue_id'], res['last_event_id'])
|
||||
|
||||
queue_id = None
|
||||
while True:
|
||||
if queue_id is None:
|
||||
(queue_id, last_event_id) = do_register()
|
||||
|
||||
res = self.get_events(queue_id=queue_id, last_event_id=last_event_id)
|
||||
if 'error' in res.get('result'):
|
||||
if res["result"] == "http-error":
|
||||
if self.verbose:
|
||||
print "HTTP error fetching events -- probably a server restart"
|
||||
elif res["result"] == "connection-error":
|
||||
if self.verbose:
|
||||
print "Connection error fetching events -- probably server is temporarily down?"
|
||||
else:
|
||||
if self.verbose:
|
||||
print "Server returned error:\n%s" % res["msg"]
|
||||
if res["msg"].startswith("Bad event queue id:"):
|
||||
# Our event queue went away, probably because
|
||||
# we were asleep or the server restarted
|
||||
# abnormally. We may have missed some
|
||||
# events while the network was down or
|
||||
# something, but there's not really anything
|
||||
# we can do about it other than resuming
|
||||
# getting new ones.
|
||||
#
|
||||
# Reset queue_id to register a new event queue.
|
||||
queue_id = None
|
||||
# TODO: Make this back off once it's more reliable
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
for event in res['events']:
|
||||
last_event_id = max(last_event_id, int(event['id']))
|
||||
callback(event)
|
||||
|
||||
def call_on_each_message(self, callback):
|
||||
def event_callback(event):
|
||||
if event['type'] == 'message':
|
||||
callback(event['message'])
|
||||
|
||||
self.call_on_each_event(event_callback, ['message'])
|
||||
|
||||
def _mk_subs(streams, **kwargs):
|
||||
result = kwargs
|
||||
result['subscriptions'] = streams
|
||||
return result
|
||||
|
||||
def _mk_rm_subs(streams):
|
||||
return {'delete': streams}
|
||||
|
||||
def _mk_deregister(queue_id):
|
||||
return {'queue_id': queue_id}
|
||||
|
||||
def _mk_events(event_types=None, narrow=[]):
|
||||
if event_types is None:
|
||||
return dict()
|
||||
return dict(event_types=event_types, narrow=narrow)
|
||||
|
||||
def _kwargs_to_dict(**kwargs):
|
||||
return kwargs
|
||||
|
||||
class ZulipStream(object):
|
||||
"""
|
||||
A Zulip stream-like object
|
||||
"""
|
||||
|
||||
def __init__(self, type, to, subject, **kwargs):
|
||||
self.client = Client(**kwargs)
|
||||
self.type = type
|
||||
self.to = to
|
||||
self.subject = subject
|
||||
|
||||
def write(self, content):
|
||||
message = {"type": self.type,
|
||||
"to": self.to,
|
||||
"subject": self.subject,
|
||||
"content": content}
|
||||
self.client.send_message(message)
|
||||
|
||||
def flush(self):
|
||||
pass
|
||||
|
||||
Client._register('send_message', url='messages', make_request=(lambda request: request))
|
||||
Client._register('update_message', method='PATCH', url='messages', make_request=(lambda request: request))
|
||||
Client._register('get_messages', method='GET', url='messages/latest', longpolling=True)
|
||||
Client._register('get_events', url='events', method='GET', longpolling=True, make_request=(lambda **kwargs: kwargs))
|
||||
Client._register('register', make_request=_mk_events)
|
||||
Client._register('deregister', url="events", method="DELETE", make_request=_mk_deregister)
|
||||
Client._register('get_profile', method='GET', url='users/me')
|
||||
Client._register('get_streams', method='GET', url='streams', make_request=_kwargs_to_dict)
|
||||
Client._register('get_members', method='GET', url='users')
|
||||
Client._register('list_subscriptions', method='GET', url='users/me/subscriptions')
|
||||
Client._register('add_subscriptions', url='users/me/subscriptions', make_request=_mk_subs)
|
||||
Client._register('remove_subscriptions', method='PATCH', url='users/me/subscriptions', make_request=_mk_rm_subs)
|
||||
Client._register('render_message', method='GET', url='messages/render')
|
||||
Client._register('create_user', method='POST', url='users')
|
||||
|
Before Width: | Height: | Size: 2.3 KiB After Width: | Height: | Size: 2.3 KiB |
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user