From 365fe0b3d5a660b5a72e33facb5ed2a1fa293bc4 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 10 Jun 2020 15:54:34 -0700 Subject: [PATCH] python: Sort imports with isort. Fixes #2665. Regenerated by tabbott with `lint --fix` after a rebase and change in parameters. Note from tabbott: In a few cases, this converts technical debt in the form of unsorted imports into different technical debt in the form of our largest files having very long, ugly import sequences at the start. I expect this change will increase pressure for us to split those files, which isn't a bad thing. Signed-off-by: Anders Kaseorg --- analytics/lib/counts.py | 34 ++- analytics/lib/fixtures.py | 1 + analytics/lib/time_utils.py | 1 + .../commands/check_analytics_state.py | 3 +- .../commands/populate_analytics_db.py | 18 +- analytics/management/commands/realm_stats.py | 13 +- analytics/management/commands/stream_stats.py | 3 +- .../commands/update_analytics_counts.py | 2 +- .../migrations/0015_clear_duplicate_counts.py | 1 + analytics/models.py | 1 + analytics/tests/test_counts.py | 66 +++-- analytics/tests/test_fixtures.py | 1 + analytics/tests/test_views.py | 21 +- analytics/views.py | 67 +++-- confirmation/migrations/0001_initial.py | 2 +- .../migrations/0002_realmcreationkey.py | 2 +- .../migrations/0005_confirmation_realm.py | 3 +- confirmation/models.py | 19 +- corporate/lib/stripe.py | 33 ++- corporate/models.py | 1 + corporate/tests/test_stripe.py | 83 ++++-- corporate/urls.py | 2 +- corporate/views.py | 47 +-- docs/conf.py | 3 +- frontend_tests/run-casper | 11 +- manage.py | 4 +- .../zulip_app_frontend/check_cron_file | 1 + .../check_send_receive_time | 11 +- .../check_postgres_replication_lag | 3 +- .../zulip_postgres_appdb/check_fts_update_log | 1 + .../files/postgresql/pg_backup_and_purge | 10 +- .../files/postgresql/process_fts_updates | 13 +- .../check_personal_zephyr_mirrors | 2 +- .../check_user_zephyr_mirror_liveness | 4 +- .../zulip_zephyr_mirror/check_zephyr_mirror | 2 +- .../files/zulip-ec2-configure-interfaces | 5 +- scripts/lib/check_rabbitmq_queue.py | 5 +- scripts/lib/clean-unused-caches | 3 +- scripts/lib/clean_emoji_cache.py | 10 +- scripts/lib/clean_node_cache.py | 10 +- scripts/lib/clean_venv_cache.py | 10 +- scripts/lib/create-production-venv | 9 +- scripts/lib/create-thumbor-venv | 9 +- scripts/lib/email-mirror-postfix | 17 +- scripts/lib/hash_reqs.py | 5 +- scripts/lib/node_cache.py | 6 +- scripts/lib/pythonrc.py | 5 +- scripts/lib/queue_workers.py | 2 +- scripts/lib/setup_path.py | 2 +- scripts/lib/setup_venv.py | 6 +- scripts/lib/sharding.py | 2 +- scripts/lib/unpack-zulip | 15 +- scripts/lib/upgrade-zulip | 20 +- scripts/lib/upgrade-zulip-from-git | 25 +- scripts/lib/upgrade-zulip-stage-2 | 14 +- scripts/lib/zulip_tools.py | 7 +- scripts/nagios/check-rabbitmq-consumers | 7 +- scripts/nagios/cron_file_helper.py | 2 +- scripts/purge-old-deployments | 5 +- scripts/restart-server | 12 +- scripts/setup/flush-memcached | 4 +- scripts/setup/generate_secrets.py | 7 +- scripts/setup/inline_email_css.py | 8 +- scripts/setup/restore-backup | 4 +- scripts/zulip-puppet-apply | 10 +- static/assets/favicon/generate | 2 +- tools/check-capitalization | 4 +- tools/check-frontend-i18n | 8 +- tools/check-issue-labels | 10 +- tools/check-provision | 8 +- tools/check-templates | 12 +- tools/check-thirdparty | 2 +- tools/create-test-api-docs | 7 +- tools/diagnose | 7 +- .../commands/crawl_with_status.py | 5 +- .../spiders/check_documentation.py | 1 - .../spiders/check_help_documentation.py | 4 +- .../spiders/common/spiders.py | 5 +- tools/droplets/add_mentor.py | 5 +- tools/droplets/create.py | 15 +- tools/fetch-contributor-data | 17 +- tools/generate-integration-docs-screenshot | 25 +- tools/i18n/process-mobile-i18n | 1 + tools/i18n/tagmessages | 3 +- tools/lib/capitalization.py | 2 +- tools/lib/gitlint-rules.py | 6 +- tools/lib/html_branches.py | 9 +- tools/lib/pretty_print.py | 9 +- tools/lib/provision.py | 29 +- tools/lib/provision_inner.py | 23 +- tools/lib/sanity_check.py | 1 + tools/lib/template_parser.py | 1 + tools/lib/test_script.py | 8 +- tools/lib/test_server.py | 6 +- tools/lint | 17 +- tools/linter_lib/pep8.py | 3 +- tools/linter_lib/pyflakes.py | 1 - tools/pretty-print-html | 6 +- tools/renumber-migrations | 7 +- tools/review | 2 +- tools/run-dev.py | 16 +- tools/run-mypy | 6 +- tools/setup/build_pygments_data | 4 +- tools/setup/emoji/build_emoji | 16 +- tools/setup/emoji/emoji_setup_utils.py | 2 - tools/setup/emoji/export_emoji_names_to_csv | 4 +- tools/setup/emoji/generate_emoji_names_table | 6 +- tools/setup/emoji/import_emoji_names_from_csv | 1 - tools/setup/emoji/test-emoji-name-scripts | 2 +- .../generate_integration_bots_avatars.py | 5 +- .../setup/generate_zulip_bots_static_files.py | 5 +- tools/show-profile-results | 2 +- tools/test-api | 9 +- tools/test-backend | 23 +- tools/test-help-documentation | 3 +- tools/test-js-with-node | 6 +- tools/test-js-with-puppeteer | 14 +- tools/test-locked-requirements | 4 +- tools/test-queue-worker-reload | 7 +- tools/test-run-dev | 2 +- tools/test-tools | 3 +- tools/tests/test_capitalization_checker.py | 7 +- tools/tests/test_check_rabbitmq_queue.py | 14 +- tools/tests/test_html_branches.py | 5 +- tools/tests/test_template_parser.py | 3 +- tools/tests/test_zulint_custom_rules.py | 8 +- tools/update-prod-static | 9 +- tools/webpack | 3 +- tools/zulip-export/zulip-export | 6 +- zerver/apps.py | 1 + zerver/context_processors.py | 30 +- zerver/data_import/gitter.py | 29 +- zerver/data_import/hipchat.py | 30 +- zerver/data_import/hipchat_attachment.py | 11 +- zerver/data_import/hipchat_user.py | 9 +- zerver/data_import/import_util.py | 23 +- zerver/data_import/mattermost.py | 41 +-- zerver/data_import/mattermost_user.py | 1 + zerver/data_import/slack.py | 70 +++-- .../data_import/slack_message_conversion.py | 2 +- zerver/decorator.py | 86 +++--- zerver/filters.py | 1 + zerver/forms.py | 60 ++-- zerver/lib/actions.py | 270 +++++++++++------- zerver/lib/addressee.py | 8 +- zerver/lib/alert_words.py | 15 +- zerver/lib/attachments.py | 4 +- zerver/lib/avatar.py | 15 +- zerver/lib/avatar_hash.py | 4 +- zerver/lib/bot_config.py | 23 +- zerver/lib/bot_lib.py | 28 +- zerver/lib/bot_storage.py | 6 +- zerver/lib/bugdown/__init__.py | 83 +++--- .../bugdown/api_arguments_table_generator.py | 9 +- .../api_return_values_table_generator.py | 5 +- zerver/lib/bugdown/fenced_code.py | 5 +- .../help_emoticon_translations_table.py | 3 +- zerver/lib/bugdown/help_relative_links.py | 3 +- zerver/lib/bugdown/help_settings_links.py | 3 +- zerver/lib/bugdown/include.py | 4 +- zerver/lib/bugdown/nested_code_blocks.py | 10 +- zerver/lib/bugdown/tabbed_sections.py | 4 +- zerver/lib/bugdown/testing_mocks.py | 2 +- zerver/lib/bulk_create.py | 11 +- zerver/lib/cache.py | 45 +-- zerver/lib/cache_helpers.py | 40 ++- zerver/lib/camo.py | 4 +- zerver/lib/ccache.py | 7 +- zerver/lib/context_managers.py | 5 +- zerver/lib/create_user.py | 24 +- zerver/lib/db.py | 7 +- zerver/lib/debug.py | 2 +- zerver/lib/dev_ldap_directory.py | 2 + zerver/lib/digest.py | 26 +- zerver/lib/display_recipient.py | 13 +- zerver/lib/domains.py | 6 +- zerver/lib/email_mirror.py | 47 +-- zerver/lib/email_mirror_helpers.py | 2 +- zerver/lib/email_notifications.py | 45 +-- zerver/lib/email_validation.py | 11 +- zerver/lib/emoji.py | 8 +- zerver/lib/error_notify.py | 9 +- zerver/lib/events.py | 58 ++-- zerver/lib/exceptions.py | 3 +- zerver/lib/export.py | 68 +++-- zerver/lib/external_accounts.py | 8 +- zerver/lib/fix_unreads.py | 4 +- zerver/lib/generate_test_data.py | 8 +- zerver/lib/hotspots.py | 6 +- zerver/lib/html_diff.py | 7 +- zerver/lib/html_to_text.py | 1 + zerver/lib/i18n.py | 13 +- zerver/lib/import_realm.py | 76 +++-- zerver/lib/initial_password.py | 7 +- zerver/lib/integrations.py | 6 +- zerver/lib/logging_util.py | 10 +- zerver/lib/management.py | 7 +- zerver/lib/mdiff.py | 3 +- zerver/lib/mention.py | 3 +- zerver/lib/message.py | 58 ++-- zerver/lib/migrate.py | 6 +- zerver/lib/mobile_auth_otp.py | 3 +- zerver/lib/name_restrictions.py | 1 + zerver/lib/narrow.py | 8 +- zerver/lib/onboarding.py | 14 +- zerver/lib/outgoing_webhook.py | 24 +- zerver/lib/parallel.py | 5 +- zerver/lib/presence.py | 15 +- zerver/lib/profile.py | 3 +- zerver/lib/push_notifications.py | 31 +- zerver/lib/pysa.py | 1 - zerver/lib/queue.py | 6 +- zerver/lib/rate_limiter.py | 10 +- zerver/lib/realm_description.py | 10 +- zerver/lib/realm_icon.py | 1 + zerver/lib/realm_logo.py | 1 + zerver/lib/redis_utils.py | 9 +- zerver/lib/remote_server.py | 5 +- zerver/lib/request.py | 12 +- zerver/lib/response.py | 6 +- zerver/lib/rest.py | 16 +- zerver/lib/retention.py | 28 +- zerver/lib/send_email.py | 32 +-- zerver/lib/server_initialization.py | 6 +- zerver/lib/sessions.py | 8 +- zerver/lib/soft_deactivation.py | 22 +- zerver/lib/sqlalchemy_utils.py | 9 +- zerver/lib/statistics.py | 5 +- zerver/lib/stream_recipient.py | 4 +- zerver/lib/stream_subscription.py | 9 +- zerver/lib/stream_topic.py | 9 +- zerver/lib/streams.py | 24 +- zerver/lib/subdomains.py | 6 +- zerver/lib/test_classes.py | 86 +++--- zerver/lib/test_fixtures.py | 20 +- zerver/lib/test_helpers.py | 51 ++-- zerver/lib/test_runner.py | 35 +-- zerver/lib/tex.py | 5 +- zerver/lib/thumbnail.py | 6 +- zerver/lib/timeout.py | 9 +- zerver/lib/timestamp.py | 3 +- zerver/lib/timezone.py | 5 +- zerver/lib/topic.py | 21 +- zerver/lib/topic_mutes.py | 25 +- zerver/lib/transfer.py | 8 +- zerver/lib/type_debug.py | 4 +- zerver/lib/types.py | 5 +- zerver/lib/unminify.py | 6 +- zerver/lib/upload.py | 70 ++--- zerver/lib/url_encoding.py | 1 + zerver/lib/url_preview/oembed.py | 6 +- zerver/lib/url_preview/parsers/__init__.py | 2 +- zerver/lib/url_preview/parsers/base.py | 1 + zerver/lib/url_preview/parsers/generic.py | 1 + zerver/lib/url_preview/parsers/open_graph.py | 1 + zerver/lib/url_preview/preview.py | 10 +- zerver/lib/user_groups.py | 7 +- zerver/lib/user_status.py | 7 +- zerver/lib/users.py | 33 ++- zerver/lib/utils.py | 4 +- zerver/lib/validator.py | 12 +- zerver/lib/webhooks/common.py | 18 +- zerver/lib/webhooks/git.py | 3 +- zerver/lib/widget.py | 5 +- zerver/lib/zcommand.py | 4 +- zerver/lib/zephyr.py | 2 + zerver/logging_handlers.py | 6 +- zerver/management/commands/check_redis.py | 3 +- .../commands/convert_gitter_data.py | 3 +- .../commands/convert_hipchat_data.py | 3 +- .../commands/convert_mattermost_data.py | 3 +- .../management/commands/convert_slack_data.py | 3 +- .../commands/generate_invite_links.py | 2 +- zerver/management/commands/import.py | 3 +- zerver/management/commands/knight.py | 2 +- .../management/commands/logout_all_users.py | 7 +- zerver/management/commands/makemessages.py | 2 +- zerver/management/commands/merge_streams.py | 7 +- .../management/commands/restore_messages.py | 7 +- zerver/management/commands/runtornado.py | 14 +- zerver/management/commands/show_admins.py | 2 + .../commands/soft_deactivate_users.py | 8 +- .../commands/transfer_uploads_to_s3.py | 3 +- zerver/middleware.py | 7 +- zerver/migrations/0030_realm_org_type.py | 1 - .../0032_verify_all_medium_avatar_images.py | 2 +- ...0038_realm_change_to_community_defaults.py | 1 - .../0060_move_avatars_to_be_uid_based.py | 1 + .../0064_sync_uploads_filesize_with_db.py | 1 + ...0077_add_file_name_field_to_realm_emoji.py | 1 + .../0086_realm_alter_default_org_type.py | 1 - .../0143_realm_bot_creation_policy.py | 2 - zerver/migrations/0185_realm_plan_type.py | 2 - .../0214_realm_invite_to_stream_policy.py | 2 - ...228_userprofile_demote_inactive_streams.py | 2 - ..._userprofile_desktop_icon_count_display.py | 2 - .../migrations/0248_userprofile_role_start.py | 2 +- .../0252_realm_user_group_edit_policy.py | 2 - .../0261_realm_private_message_policy.py | 2 - .../migrations/0262_mutedtopic_date_muted.py | 2 +- .../0263_stream_stream_post_policy.py | 2 - zerver/migrations/0266_userpresence_realm.py | 3 +- ..._add_userpresence_realm_timestamp_index.py | 3 +- zerver/migrations/0270_huddle_recipient.py | 3 +- .../0273_migrate_old_bot_messages.py | 3 +- zerver/migrations/0276_alertword.py | 2 +- zerver/migrations/0277_migrate_alert_word.py | 6 +- ...84_convert_realm_admins_to_realm_owners.py | 3 +- zerver/models.py | 132 ++++++--- zerver/openapi/curl_param_value_generators.py | 9 +- zerver/openapi/javascript_examples.py | 4 +- zerver/openapi/markdown_extension.py | 11 +- zerver/openapi/python_examples.py | 12 +- zerver/openapi/test_curl_examples.py | 13 +- zerver/signals.py | 5 +- zerver/templatetags/app_filters.py | 14 +- zerver/tests/test_alert_words.py | 30 +- zerver/tests/test_archive.py | 11 +- zerver/tests/test_attachments.py | 3 +- zerver/tests/test_audit_log.py | 38 ++- zerver/tests/test_auth_backends.py | 155 ++++++---- zerver/tests/test_bots.py | 36 ++- zerver/tests/test_bugdown.py | 51 ++-- zerver/tests/test_cache.py | 31 +- zerver/tests/test_camo.py | 1 + zerver/tests/test_compatibility.py | 2 +- zerver/tests/test_create_video_call.py | 1 + zerver/tests/test_custom_profile_data.py | 30 +- zerver/tests/test_decorators.py | 94 +++--- zerver/tests/test_digest.py | 24 +- zerver/tests/test_docs.py | 11 +- zerver/tests/test_email_change.py | 19 +- zerver/tests/test_email_log.py | 2 + zerver/tests/test_email_mirror.py | 65 ++--- zerver/tests/test_email_notifications.py | 28 +- zerver/tests/test_embedded_bot_system.py | 10 +- zerver/tests/test_event_queue.py | 24 +- zerver/tests/test_events.py | 115 +++++--- zerver/tests/test_external.py | 22 +- zerver/tests/test_gitter_importer.py | 27 +- zerver/tests/test_hipchat_importer.py | 18 +- zerver/tests/test_home.py | 34 ++- zerver/tests/test_hotspots.py | 7 +- zerver/tests/test_i18n.py | 19 +- zerver/tests/test_import_export.py | 99 +++---- zerver/tests/test_integrations.py | 14 +- zerver/tests/test_integrations_dev_panel.py | 7 +- zerver/tests/test_internet.py | 5 +- zerver/tests/test_legacy_subject.py | 5 +- zerver/tests/test_link_embed.py | 20 +- zerver/tests/test_logging_handlers.py | 10 +- zerver/tests/test_management_commands.py | 28 +- zerver/tests/test_mattermost_importer.py | 45 +-- .../tests/test_message_edit_notifications.py | 20 +- zerver/tests/test_messages.py | 117 ++++---- zerver/tests/test_middleware.py | 4 +- zerver/tests/test_migrations.py | 3 +- zerver/tests/test_muting.py | 16 +- zerver/tests/test_narrow.py | 82 +++--- zerver/tests/test_new_users.py | 12 +- zerver/tests/test_onboarding.py | 7 +- zerver/tests/test_openapi.py | 47 ++- .../tests/test_outgoing_webhook_interfaces.py | 19 +- zerver/tests/test_outgoing_webhook_system.py | 22 +- zerver/tests/test_populate_db.py | 7 +- zerver/tests/test_presence.py | 23 +- zerver/tests/test_push_notifications.py | 78 ++--- zerver/tests/test_queue.py | 8 +- zerver/tests/test_queue_worker.py | 17 +- zerver/tests/test_rate_limiter.py | 14 +- zerver/tests/test_reactions.py | 12 +- zerver/tests/test_realm.py | 39 +-- zerver/tests/test_realm_domains.py | 18 +- zerver/tests/test_realm_emoji.py | 4 +- zerver/tests/test_realm_export.py | 22 +- zerver/tests/test_realm_filters.py | 1 + zerver/tests/test_redis_utils.py | 11 +- zerver/tests/test_report.py | 9 +- zerver/tests/test_retention.py | 40 ++- zerver/tests/test_service_bot_system.py | 21 +- zerver/tests/test_sessions.py | 19 +- zerver/tests/test_settings.py | 14 +- zerver/tests/test_signup.py | 173 ++++++----- zerver/tests/test_slack_importer.py | 87 +++--- zerver/tests/test_slack_message_conversion.py | 14 +- zerver/tests/test_soft_deactivation.py | 13 +- zerver/tests/test_subdomains.py | 3 +- zerver/tests/test_submessage.py | 13 +- zerver/tests/test_subs.py | 136 ++++----- zerver/tests/test_templates.py | 4 +- zerver/tests/test_thumbnail.py | 13 +- zerver/tests/test_timestamp.py | 18 +- zerver/tests/test_tornado.py | 14 +- zerver/tests/test_transfer.py | 26 +- zerver/tests/test_tutorial.py | 4 +- zerver/tests/test_type_debug.py | 5 +- zerver/tests/test_typing.py | 20 +- zerver/tests/test_unread.py | 29 +- zerver/tests/test_upload.py | 107 ++++--- zerver/tests/test_urls.py | 7 +- zerver/tests/test_user_groups.py | 17 +- zerver/tests/test_user_status.py | 24 +- zerver/tests/test_users.py | 84 +++--- zerver/tests/test_webhooks_common.py | 20 +- zerver/tests/test_widgets.py | 12 +- zerver/tests/test_zcommand.py | 5 +- zerver/tests/test_zephyr.py | 10 +- zerver/tornado/application.py | 3 +- zerver/tornado/autoreload.py | 9 +- zerver/tornado/descriptors.py | 2 +- zerver/tornado/event_queue.py | 68 +++-- zerver/tornado/exceptions.py | 1 + zerver/tornado/handlers.py | 2 +- zerver/tornado/sharding.py | 6 +- zerver/tornado/views.py | 18 +- zerver/views/alert_words.py | 12 +- zerver/views/archive.py | 14 +- zerver/views/attachments.py | 5 +- zerver/views/auth.py | 83 +++--- zerver/views/camo.py | 8 +- zerver/views/compatibility.py | 5 +- zerver/views/custom_profile_fields.py | 44 +-- zerver/views/development/email_log.py | 32 +-- zerver/views/development/integrations.py | 12 +- zerver/views/development/registration.py | 9 +- zerver/views/digest.py | 8 +- zerver/views/documentation.py | 22 +- zerver/views/email_mirror.py | 8 +- zerver/views/events_register.py | 6 +- zerver/views/home.py | 33 +-- zerver/views/hotspots.py | 3 +- zerver/views/invite.py | 32 ++- zerver/views/messages.py | 135 ++++++--- zerver/views/muting.py | 12 +- zerver/views/pointer.py | 3 +- zerver/views/portico.py | 5 +- zerver/views/presence.py | 21 +- zerver/views/push_notifications.py | 10 +- zerver/views/reactions.py | 6 +- zerver/views/realm.py | 39 ++- zerver/views/realm_domains.py | 5 +- zerver/views/realm_emoji.py | 10 +- zerver/views/realm_export.py | 13 +- zerver/views/realm_filters.py | 6 +- zerver/views/realm_icon.py | 2 +- zerver/views/realm_logo.py | 6 +- zerver/views/registration.py | 113 +++++--- zerver/views/report.py | 11 +- zerver/views/storage.py | 16 +- zerver/views/streams.py | 96 +++++-- zerver/views/submessage.py | 12 +- zerver/views/thumbnail.py | 11 +- zerver/views/tutorial.py | 3 +- zerver/views/typing.py | 6 +- zerver/views/unsubscribe.py | 13 +- zerver/views/upload.py | 27 +- zerver/views/user_groups.py | 34 ++- zerver/views/user_settings.py | 54 ++-- zerver/views/users.py | 94 ++++-- zerver/views/video_calls.py | 6 +- zerver/views/zephyr.py | 25 +- zerver/webhooks/alertmanager/view.py | 1 + zerver/webhooks/ansibletower/view.py | 3 +- zerver/webhooks/appveyor/view.py | 3 +- zerver/webhooks/basecamp/view.py | 3 +- zerver/webhooks/beanstalk/tests.py | 1 - zerver/webhooks/beanstalk/view.py | 3 +- zerver/webhooks/bitbucket/tests.py | 1 - zerver/webhooks/bitbucket/view.py | 3 +- zerver/webhooks/bitbucket2/view.py | 23 +- zerver/webhooks/bitbucket3/view.py | 25 +- zerver/webhooks/clubhouse/tests.py | 1 - zerver/webhooks/clubhouse/view.py | 3 +- zerver/webhooks/flock/view.py | 3 +- zerver/webhooks/github/view.py | 27 +- zerver/webhooks/gitlab/view.py | 22 +- zerver/webhooks/gogs/view.py | 24 +- zerver/webhooks/gosquared/view.py | 3 +- zerver/webhooks/grafana/view.py | 2 +- zerver/webhooks/groove/view.py | 9 +- zerver/webhooks/harbor/view.py | 3 +- zerver/webhooks/hellosign/tests.py | 4 +- zerver/webhooks/hellosign/view.py | 2 +- zerver/webhooks/insping/view.py | 3 +- zerver/webhooks/intercom/view.py | 3 +- zerver/webhooks/jira/view.py | 3 +- zerver/webhooks/netlify/view.py | 9 +- zerver/webhooks/newrelic/view.py | 3 +- zerver/webhooks/pagerduty/view.py | 3 +- zerver/webhooks/pingdom/view.py | 3 +- zerver/webhooks/pivotal/view.py | 3 +- zerver/webhooks/raygun/view.py | 3 +- zerver/webhooks/reviewboard/view.py | 9 +- zerver/webhooks/semaphore/tests.py | 4 +- zerver/webhooks/semaphore/view.py | 3 +- zerver/webhooks/sentry/view.py | 3 +- zerver/webhooks/slack_incoming/view.py | 9 +- zerver/webhooks/statuspage/view.py | 3 +- zerver/webhooks/stripe/view.py | 3 +- zerver/webhooks/teamcity/tests.py | 3 +- zerver/webhooks/teamcity/view.py | 6 +- zerver/webhooks/transifex/view.py | 3 +- zerver/webhooks/trello/view/__init__.py | 9 +- zerver/webhooks/updown/view.py | 3 +- zerver/webhooks/zabbix/view.py | 3 +- zerver/worker/queue_processors.py | 126 +++++--- zilencer/forms.py | 1 + .../commands/add_mock_conversation.py | 12 +- zilencer/management/commands/add_new_realm.py | 3 +- zilencer/management/commands/populate_db.py | 57 ++-- zilencer/models.py | 3 +- zilencer/views.py | 39 ++- zproject/backends.py | 88 +++--- zproject/computed_settings.py | 19 +- zproject/config.py | 2 +- zproject/configured_settings.py | 1 + zproject/default_settings.py | 5 +- zproject/dev_settings.py | 1 + zproject/dev_urls.py | 8 +- zproject/email_backends.py | 9 +- zproject/jinja2/__init__.py | 2 +- zproject/legacy_urls.py | 5 +- zproject/test_extra_settings.py | 1 + zproject/test_settings.py | 1 + zproject/urls.py | 64 +++-- zproject/wsgi.py | 3 + zthumbor/loaders/helpers.py | 2 +- zthumbor/loaders/zloader.py | 28 +- 528 files changed, 4801 insertions(+), 3806 deletions(-) diff --git a/analytics/lib/counts.py b/analytics/lib/counts.py index f83346a8e1..a3f9ee8406 100644 --- a/analytics/lib/counts.py +++ b/analytics/lib/counts.py @@ -1,23 +1,35 @@ +import logging import time from collections import OrderedDict, defaultdict from datetime import datetime, timedelta -import logging -from typing import Callable, Dict, List, \ - Optional, Tuple, Type, Union +from typing import Callable, Dict, List, Optional, Tuple, Type, Union from django.conf import settings from django.db import connection from django.db.models import F -from psycopg2.sql import Composable, Identifier, Literal, SQL +from psycopg2.sql import SQL, Composable, Identifier, Literal -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch, last_successful_fill +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) from zerver.lib.logging_util import log_to_file -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, floor_to_hour, verify_UTC -from zerver.models import Message, Realm, RealmAuditLog, \ - Stream, UserActivityInterval, UserProfile, models +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, floor_to_hour, verify_UTC +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Stream, + UserActivityInterval, + UserProfile, + models, +) ## Logging setup ## diff --git a/analytics/lib/fixtures.py b/analytics/lib/fixtures.py index a439b56d95..c707220a19 100644 --- a/analytics/lib/fixtures.py +++ b/analytics/lib/fixtures.py @@ -4,6 +4,7 @@ from typing import List from analytics.lib.counts import CountStat + def generate_time_series_data(days: int=100, business_hours_base: float=10, non_business_hours_base: float=10, growth: float=1, autocorrelation: float=0, spikiness: float=1, diff --git a/analytics/lib/time_utils.py b/analytics/lib/time_utils.py index 48d8f54223..b990ae252d 100644 --- a/analytics/lib/time_utils.py +++ b/analytics/lib/time_utils.py @@ -4,6 +4,7 @@ from typing import List, Optional from analytics.lib.counts import CountStat from zerver.lib.timestamp import floor_to_day, floor_to_hour, verify_UTC + # If min_length is None, returns end_times from ceiling(start) to floor(end), inclusive. # If min_length is greater than 0, pads the list to the left. # So informally, time_range(Sep 20, Sep 22, day, None) returns [Sep 20, Sep 21, Sep 22], diff --git a/analytics/management/commands/check_analytics_state.py b/analytics/management/commands/check_analytics_state.py index c1a14f0341..698317b6f5 100644 --- a/analytics/management/commands/check_analytics_state.py +++ b/analytics/management/commands/check_analytics_state.py @@ -8,8 +8,7 @@ from django.utils.timezone import now as timezone_now from analytics.lib.counts import COUNT_STATS, CountStat from analytics.models import installation_epoch, last_successful_fill -from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, \ - floor_to_hour, verify_UTC +from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day, floor_to_hour, verify_UTC from zerver.models import Realm states = { diff --git a/analytics/management/commands/populate_analytics_db.py b/analytics/management/commands/populate_analytics_db.py index 462f41447b..70857dc630 100644 --- a/analytics/management/commands/populate_analytics_db.py +++ b/analytics/management/commands/populate_analytics_db.py @@ -1,21 +1,25 @@ from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Type - from unittest import mock + from django.core.management.base import BaseCommand from django.utils.timezone import now as timezone_now -from analytics.lib.counts import COUNT_STATS, CountStat, \ - do_drop_all_analytics_tables +from analytics.lib.counts import COUNT_STATS, CountStat, do_drop_all_analytics_tables from analytics.lib.fixtures import generate_time_series_data from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, FillState, InstallationCount, \ - RealmCount, StreamCount, UserCount +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, +) from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, do_change_user_role from zerver.lib.create_user import create_user from zerver.lib.timestamp import floor_to_day -from zerver.models import Client, Realm, Recipient, Stream, Subscription, \ - UserProfile +from zerver.models import Client, Realm, Recipient, Stream, Subscription, UserProfile class Command(BaseCommand): diff --git a/analytics/management/commands/realm_stats.py b/analytics/management/commands/realm_stats.py index 8730a3a217..336a092f6c 100644 --- a/analytics/management/commands/realm_stats.py +++ b/analytics/management/commands/realm_stats.py @@ -6,8 +6,17 @@ from django.core.management.base import BaseCommand, CommandError from django.db.models import Count from django.utils.timezone import now as timezone_now -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - UserActivity, UserMessage, UserProfile, get_realm +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + get_realm, +) MOBILE_CLIENT_LIST = ["Android", "ios"] HUMAN_CLIENT_LIST = MOBILE_CLIENT_LIST + ["website"] diff --git a/analytics/management/commands/stream_stats.py b/analytics/management/commands/stream_stats.py index b6bde4bdc2..4bb861c00c 100644 --- a/analytics/management/commands/stream_stats.py +++ b/analytics/management/commands/stream_stats.py @@ -4,8 +4,7 @@ from typing import Any from django.core.management.base import BaseCommand, CommandError from django.db.models import Q -from zerver.models import Message, Realm, Recipient, Stream, Subscription, \ - get_realm +from zerver.models import Message, Realm, Recipient, Stream, Subscription, get_realm class Command(BaseCommand): diff --git a/analytics/management/commands/update_analytics_counts.py b/analytics/management/commands/update_analytics_counts.py index 9e89bb2246..2f66409563 100644 --- a/analytics/management/commands/update_analytics_counts.py +++ b/analytics/management/commands/update_analytics_counts.py @@ -1,8 +1,8 @@ import os import time from argparse import ArgumentParser -from typing import Any, Dict from datetime import timezone +from typing import Any, Dict from django.conf import settings from django.core.management.base import BaseCommand diff --git a/analytics/migrations/0015_clear_duplicate_counts.py b/analytics/migrations/0015_clear_duplicate_counts.py index b97f8265fa..9533f21bde 100644 --- a/analytics/migrations/0015_clear_duplicate_counts.py +++ b/analytics/migrations/0015_clear_duplicate_counts.py @@ -3,6 +3,7 @@ from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models import Count, Sum + def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """This is a preparatory migration for our Analytics tables. diff --git a/analytics/models.py b/analytics/models.py index b81ea5cff1..20bcc8b6a6 100644 --- a/analytics/models.py +++ b/analytics/models.py @@ -7,6 +7,7 @@ from django.db.models import Q, UniqueConstraint from zerver.lib.timestamp import floor_to_day from zerver.models import Realm, Stream, UserProfile + class FillState(models.Model): property: str = models.CharField(max_length=40, unique=True) end_time: datetime.datetime = models.DateTimeField() diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index 7992da7af6..52f58f6f5a 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple, Type - from unittest import mock + import ujson from django.apps import apps from django.db import models @@ -10,24 +10,58 @@ from django.test import TestCase from django.utils.timezone import now as timezone_now from psycopg2.sql import SQL, Literal -from analytics.lib.counts import COUNT_STATS, CountStat, get_count_stats, \ - DependentCountStat, LoggingCountStat, do_aggregate_to_summary_table, \ - do_drop_all_analytics_tables, do_drop_single_stat, \ - do_fill_count_stat_at_hour, do_increment_logging_stat, \ - process_count_stat, sql_data_collector -from analytics.models import BaseCount, \ - FillState, InstallationCount, RealmCount, StreamCount, \ - UserCount, installation_epoch -from zerver.lib.actions import do_activate_user, do_create_user, \ - do_deactivate_user, do_reactivate_user, update_user_activity_interval, \ - do_invite_users, do_revoke_user_invite, do_resend_user_invite_email, \ - InvitationError +from analytics.lib.counts import ( + COUNT_STATS, + CountStat, + DependentCountStat, + LoggingCountStat, + do_aggregate_to_summary_table, + do_drop_all_analytics_tables, + do_drop_single_stat, + do_fill_count_stat_at_hour, + do_increment_logging_stat, + get_count_stats, + process_count_stat, + sql_data_collector, +) +from analytics.models import ( + BaseCount, + FillState, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, +) +from zerver.lib.actions import ( + InvitationError, + do_activate_user, + do_create_user, + do_deactivate_user, + do_invite_users, + do_reactivate_user, + do_resend_user_invite_email, + do_revoke_user_invite, + update_user_activity_interval, +) from zerver.lib.create_user import create_user from zerver.lib.timestamp import TimezoneNotUTCException, floor_to_day from zerver.lib.topic import DB_TOPIC_NAME -from zerver.models import Client, Huddle, Message, Realm, \ - RealmAuditLog, Recipient, Stream, UserActivityInterval, \ - UserProfile, get_client, get_user, PreregistrationUser +from zerver.models import ( + Client, + Huddle, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + Recipient, + Stream, + UserActivityInterval, + UserProfile, + get_client, + get_user, +) + class AnalyticsTestCase(TestCase): MINUTE = timedelta(seconds = 60) diff --git a/analytics/tests/test_fixtures.py b/analytics/tests/test_fixtures.py index d1e31142e6..a5e7f47125 100644 --- a/analytics/tests/test_fixtures.py +++ b/analytics/tests/test_fixtures.py @@ -2,6 +2,7 @@ from analytics.lib.counts import CountStat from analytics.lib.fixtures import generate_time_series_data from zerver.lib.test_classes import ZulipTestCase + # A very light test suite; the code being tested is not run in production. class TestFixtures(ZulipTestCase): def test_deterministic_settings(self) -> None: diff --git a/analytics/tests/test_views.py b/analytics/tests/test_views.py index 1b8d1465aa..2ec434b456 100644 --- a/analytics/tests/test_views.py +++ b/analytics/tests/test_views.py @@ -1,24 +1,21 @@ from datetime import datetime, timedelta, timezone from typing import List, Optional - from unittest import mock -from django.http import HttpResponse + import ujson +from django.http import HttpResponse from django.utils.timezone import now as timezone_now from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import FillState, \ - RealmCount, UserCount, last_successful_fill -from analytics.views import rewrite_client_arrays, \ - sort_by_totals, sort_client_labels -from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from analytics.models import FillState, RealmCount, UserCount, last_successful_fill +from analytics.views import rewrite_client_arrays, sort_by_totals, sort_client_labels +from zerver.lib.actions import do_create_multiuse_invite_link, do_send_realm_reactivation_email from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import ceiling_to_day, \ - ceiling_to_hour, datetime_to_timestamp -from zerver.lib.actions import do_create_multiuse_invite_link, \ - do_send_realm_reactivation_email -from zerver.models import Client, get_realm, MultiuseInvite +from zerver.lib.test_helpers import reset_emails_in_zulip_realm +from zerver.lib.timestamp import ceiling_to_day, ceiling_to_hour, datetime_to_timestamp +from zerver.models import Client, MultiuseInvite, get_realm + class TestStatsEndpoint(ZulipTestCase): def test_stats(self) -> None: diff --git a/analytics/views.py b/analytics/views.py index 702bb34dc6..3444db001e 100644 --- a/analytics/views.py +++ b/analytics/views.py @@ -6,54 +6,75 @@ import urllib from collections import defaultdict from datetime import datetime, timedelta, timezone from decimal import Decimal - -from typing import Any, Callable, Dict, List, \ - Optional, Set, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union import pytz from django.conf import settings -from django.urls import reverse +from django.core.exceptions import ValidationError +from django.core.validators import URLValidator from django.db import connection from django.db.models.query import QuerySet from django.http import HttpRequest, HttpResponse, HttpResponseNotFound from django.shortcuts import render from django.template import loader +from django.urls import reverse +from django.utils.timesince import timesince from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.utils.timesince import timesince -from django.core.validators import URLValidator -from django.core.exceptions import ValidationError from jinja2 import Markup as mark_safe -from psycopg2.sql import Composable, Literal, SQL +from psycopg2.sql import SQL, Composable, Literal from analytics.lib.counts import COUNT_STATS, CountStat from analytics.lib.time_utils import time_range -from analytics.models import BaseCount, InstallationCount, \ - RealmCount, StreamCount, UserCount, last_successful_fill, installation_epoch -from confirmation.models import Confirmation, confirmation_url, _properties -from zerver.decorator import require_server_admin, require_server_admin_api, \ - to_utc_datetime, zulip_login_required, require_non_guest_user +from analytics.models import ( + BaseCount, + InstallationCount, + RealmCount, + StreamCount, + UserCount, + installation_epoch, + last_successful_fill, +) +from confirmation.models import Confirmation, _properties, confirmation_url +from confirmation.settings import STATUS_ACTIVE +from zerver.decorator import ( + require_non_guest_user, + require_server_admin, + require_server_admin_api, + to_utc_datetime, + zulip_login_required, +) +from zerver.lib.actions import ( + do_change_plan_type, + do_deactivate_realm, + do_scrub_realm, + do_send_realm_reactivation_email, +) from zerver.lib.exceptions import JsonableError +from zerver.lib.realm_icon import realm_icon_url from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime -from zerver.lib.realm_icon import realm_icon_url -from zerver.views.invite import get_invitee_emails_set from zerver.lib.subdomains import get_subdomain_from_hostname -from zerver.lib.actions import do_change_plan_type, do_deactivate_realm, \ - do_send_realm_reactivation_email, do_scrub_realm +from zerver.lib.timestamp import convert_to_UTC, timestamp_to_datetime from zerver.lib.validator import to_non_negative_int -from confirmation.settings import STATUS_ACTIVE +from zerver.views.invite import get_invitee_emails_set if settings.BILLING_ENABLED: from corporate.lib.stripe import attach_discount_to_realm, get_discount_for_realm -from zerver.models import Client, get_realm, Realm, UserActivity, UserActivityInterval, \ - UserProfile, PreregistrationUser, MultiuseInvite +from zerver.models import ( + Client, + MultiuseInvite, + PreregistrationUser, + Realm, + UserActivity, + UserActivityInterval, + UserProfile, + get_realm, +) if settings.ZILENCER_ENABLED: - from zilencer.models import RemoteInstallationCount, RemoteRealmCount, \ - RemoteZulipServer + from zilencer.models import RemoteInstallationCount, RemoteRealmCount, RemoteZulipServer else: from unittest.mock import Mock RemoteInstallationCount = Mock() # type: ignore[misc] # https://github.com/JukkaL/mypy/issues/1188 diff --git a/confirmation/migrations/0001_initial.py b/confirmation/migrations/0001_initial.py index af6fed33e7..75b8fb2f97 100644 --- a/confirmation/migrations/0001_initial.py +++ b/confirmation/migrations/0001_initial.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0002_realmcreationkey.py b/confirmation/migrations/0002_realmcreationkey.py index 2053e28340..fac60f8186 100644 --- a/confirmation/migrations/0002_realmcreationkey.py +++ b/confirmation/migrations/0002_realmcreationkey.py @@ -1,5 +1,5 @@ -from django.db import models, migrations import django.utils.timezone +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/migrations/0005_confirmation_realm.py b/confirmation/migrations/0005_confirmation_realm.py index 2dc1088e9d..75179cb797 100644 --- a/confirmation/migrations/0005_confirmation_realm.py +++ b/confirmation/migrations/0005_confirmation_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.6 on 2017-11-30 00:13 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/confirmation/models.py b/confirmation/models.py index c958e57584..8471ed8940 100644 --- a/confirmation/models.py +++ b/confirmation/models.py @@ -1,24 +1,23 @@ # Copyright: (c) 2008, Jarek Zgoda __revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $' - import datetime +import string +from random import SystemRandom +from typing import Dict, Optional, Union +from django.conf import settings +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models import CASCADE -from django.urls import reverse -from django.conf import settings -from django.contrib.contenttypes.models import ContentType -from django.contrib.contenttypes.fields import GenericForeignKey from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.urls import reverse from django.utils.timezone import now as timezone_now -from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite, \ - UserProfile, Realm -from random import SystemRandom -import string -from typing import Dict, Optional, Union +from zerver.models import EmailChangeStatus, MultiuseInvite, PreregistrationUser, Realm, UserProfile + class ConfirmationKeyException(Exception): WRONG_LENGTH = 1 diff --git a/corporate/lib/stripe.py b/corporate/lib/stripe.py index 54ee6d3610..2eabbcabc3 100644 --- a/corporate/lib/stripe.py +++ b/corporate/lib/stripe.py @@ -1,26 +1,31 @@ -from datetime import datetime, timedelta -from decimal import Decimal -from functools import wraps import logging import math import os -from typing import Any, Callable, Dict, Optional, TypeVar, Tuple, cast -import ujson +from datetime import datetime, timedelta +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, cast -from django.conf import settings -from django.db import transaction -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now -from django.core.signing import Signer import stripe +import ujson +from django.conf import settings +from django.core.signing import Signer +from django.db import transaction +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) from zerver.lib.logging_util import log_to_file from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.utils import generate_random_token -from zerver.models import Realm, UserProfile, RealmAuditLog -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_current_plan_by_customer, get_customer_by_realm, \ - get_current_plan_by_realm +from zerver.models import Realm, RealmAuditLog, UserProfile from zproject.config import get_secret STRIPE_PUBLISHABLE_KEY = get_secret('stripe_publishable_key') diff --git a/corporate/models.py b/corporate/models.py index bb6ea75fe7..715f10d7e3 100644 --- a/corporate/models.py +++ b/corporate/models.py @@ -7,6 +7,7 @@ from django.db.models import CASCADE from zerver.models import Realm + class Customer(models.Model): realm: Realm = models.OneToOneField(Realm, on_delete=CASCADE) stripe_customer_id: str = models.CharField(max_length=255, null=True, unique=True) diff --git a/corporate/tests/test_stripe.py b/corporate/tests/test_stripe.py index 1be9d5be65..cfd9586917 100644 --- a/corporate/tests/test_stripe.py +++ b/corporate/tests/test_stripe.py @@ -1,43 +1,66 @@ -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from functools import wraps -from unittest.mock import Mock, patch +import json import operator import os import re import sys -from typing import Any, Callable, Dict, List, Optional, TypeVar, Tuple, cast -import ujson -import json -import responses +from datetime import datetime, timedelta, timezone +from decimal import Decimal +from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast +from unittest.mock import Mock, patch -from django.core import signing -from django.urls.resolvers import get_resolver -from django.http import HttpResponse +import responses +import stripe +import ujson from django.conf import settings +from django.core import signing +from django.http import HttpResponse +from django.urls.resolvers import get_resolver from django.utils.timezone import now as timezone_now -import stripe - -from zerver.lib.actions import do_deactivate_user, do_create_user, \ - do_activate_user, do_reactivate_user, do_deactivate_realm, \ - do_reactivate_realm +from corporate.lib.stripe import ( + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + BillingError, + StripeCardError, + add_months, + attach_discount_to_realm, + catch_stripe_errors, + compute_plan_parameters, + get_discount_for_realm, + get_latest_seat_count, + invoice_plan, + invoice_plans_as_needed, + make_end_of_cycle_updates_if_needed, + next_month, + process_initial_upgrade, + sign_string, + stripe_get_customer, + unsign_string, + update_license_ledger_for_automanaged_plan, + update_license_ledger_if_needed, + update_or_create_stripe_customer, +) +from corporate.models import ( + Customer, + CustomerPlan, + LicenseLedger, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.lib.actions import ( + do_activate_user, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import reset_emails_in_zulip_realm -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.models import Realm, UserProfile, get_realm, RealmAuditLog -from corporate.lib.stripe import catch_stripe_errors, attach_discount_to_realm, \ - get_latest_seat_count, sign_string, unsign_string, \ - BillingError, StripeCardError, stripe_get_customer, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, \ - add_months, next_month, \ - compute_plan_parameters, update_or_create_stripe_customer, \ - process_initial_upgrade, make_end_of_cycle_updates_if_needed, \ - update_license_ledger_if_needed, update_license_ledger_for_automanaged_plan, \ - invoice_plan, invoice_plans_as_needed, get_discount_for_realm -from corporate.models import Customer, CustomerPlan, LicenseLedger, \ - get_customer_by_realm, get_current_plan_by_customer, \ - get_current_plan_by_realm +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, RealmAuditLog, UserProfile, get_realm CallableT = TypeVar('CallableT', bound=Callable[..., Any]) diff --git a/corporate/urls.py b/corporate/urls.py index c799f7edf2..4ebb7c2e6b 100644 --- a/corporate/urls.py +++ b/corporate/urls.py @@ -1,8 +1,8 @@ from typing import Any -from django.views.generic import TemplateView from django.conf.urls import include from django.urls import path +from django.views.generic import TemplateView import corporate.views from zerver.lib.rest import rest_dispatch diff --git a/corporate/views.py b/corporate/views.py index 0f8c8f3ffc..57f6e8d33b 100644 --- a/corporate/views.py +++ b/corporate/views.py @@ -1,30 +1,45 @@ import logging from decimal import Decimal -import stripe -from typing import Any, Dict, cast, Optional, Union +from typing import Any, Dict, Optional, Union, cast +import stripe +from django.conf import settings from django.core import signing from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.utils.timezone import now as timezone_now -from django.utils.translation import ugettext as _ from django.shortcuts import render from django.urls import reverse -from django.conf import settings +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ -from zerver.decorator import zulip_login_required, require_billing_access +from corporate.lib.stripe import ( + DEFAULT_INVOICE_DAYS_UNTIL_DUE, + MAX_INVOICED_LICENSES, + MIN_INVOICED_LICENSES, + STRIPE_PUBLISHABLE_KEY, + BillingError, + do_change_plan_status, + do_replace_payment_source, + downgrade_now, + get_latest_seat_count, + make_end_of_cycle_updates_if_needed, + process_initial_upgrade, + renewal_amount, + sign_string, + start_of_next_billing_cycle, + stripe_get_customer, + unsign_string, +) +from corporate.models import ( + CustomerPlan, + get_current_plan_by_customer, + get_current_plan_by_realm, + get_customer_by_realm, +) +from zerver.decorator import require_billing_access, zulip_login_required from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_string, check_int +from zerver.lib.validator import check_int, check_string from zerver.models import UserProfile -from corporate.lib.stripe import STRIPE_PUBLISHABLE_KEY, \ - stripe_get_customer, get_latest_seat_count, \ - process_initial_upgrade, sign_string, \ - unsign_string, BillingError, do_change_plan_status, do_replace_payment_source, \ - MIN_INVOICED_LICENSES, MAX_INVOICED_LICENSES, DEFAULT_INVOICE_DAYS_UNTIL_DUE, \ - start_of_next_billing_cycle, renewal_amount, \ - make_end_of_cycle_updates_if_needed, downgrade_now -from corporate.models import CustomerPlan, get_current_plan_by_customer, \ - get_customer_by_realm, get_current_plan_by_realm billing_logger = logging.getLogger('corporate.stripe') diff --git a/docs/conf.py b/docs/conf.py index f11bad4cc1..45f49e78bd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,10 +10,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. - import os import sys - from typing import Any, Dict, List, Optional # If extensions (or modules to document with autodoc) are in another directory, @@ -310,6 +308,7 @@ source_suffix = { # See https://github.com/zulip/zulip/issues/13263 for details. from recommonmark.parser import CommonMarkParser + class CustomCommonMarkParser(CommonMarkParser): def visit_document(self, node): pass diff --git a/frontend_tests/run-casper b/frontend_tests/run-casper index e59ad4f3b6..0261bdf8c4 100755 --- a/frontend_tests/run-casper +++ b/frontend_tests/run-casper @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import argparse +import glob +import os +import shlex import subprocess import sys -import os -import glob -import shlex # # In order to use remote casperjs debugging, pass the --remote-debug flag @@ -64,13 +64,14 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) +from typing import Iterable, List + from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files from tools.lib.test_server import test_server_running -from typing import Iterable, List - assert_provisioning_status_ok(options.force) os.chdir(ZULIP_PATH) diff --git a/manage.py b/manage.py index ce8204c393..30f8fb0e7b 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +import configparser import os import sys -import configparser + if sys.version_info <= (3, 0): print("Error: Zulip is a Python 3 project, and cannot be run with Python 2.") print("Use e.g. `/path/to/manage.py` not `python /path/to/manage.py`.") @@ -37,6 +38,7 @@ if __name__ == "__main__": from django.conf import settings from django.core.management import execute_from_command_line from django.core.management.base import CommandError + from scripts.lib.zulip_tools import log_management_command log_management_command(" ".join(sys.argv), settings.MANAGEMENT_LOG_PATH) diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file index 64f53c11e7..86df41348a 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_cron_file @@ -6,6 +6,7 @@ import sys import time from typing import Tuple + def nagios_from_file(results_file: str, max_time_diff: int=60 * 2) -> 'Tuple[int, str]': """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time index 739a78c7a4..67347bfa45 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time +++ b/puppet/zulip/files/nagios_plugins/zulip_app_frontend/check_send_receive_time @@ -8,12 +8,12 @@ It supports both munin and nagios outputs It must be run on a machine that is using the live database for the Django ORM. """ -import sys import argparse +import os import random +import sys import time import traceback -import os sys.path.append('.') sys.path.append('/home/zulip/deployments/current') @@ -21,10 +21,10 @@ from scripts.lib.setup_path import setup_path setup_path() -import django - from typing import Any, Dict, List, Optional +import django + usage = """Usage: send-receive.py [options] [config] 'config' is optional, if present will return config info. @@ -75,9 +75,10 @@ os.environ['DJANGO_SETTINGS_MODULE'] = "zproject.settings" django.setup() -from zerver.models import get_system_bot from django.conf import settings +from zerver.models import get_system_bot + states = { "OK": 0, "WARNING": 1, diff --git a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag index 32dd089e05..f209cc32e1 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag +++ b/puppet/zulip/files/nagios_plugins/zulip_nagios_server/check_postgres_replication_lag @@ -4,10 +4,9 @@ Nagios plugin to check the difference between the primary and secondary Postgres servers' xlog location. """ - import configparser -import subprocess import re +import subprocess from typing import NoReturn states = { diff --git a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log index 9b19f5cbc6..d5665b98e1 100755 --- a/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log +++ b/puppet/zulip/files/nagios_plugins/zulip_postgres_appdb/check_fts_update_log @@ -4,6 +4,7 @@ Nagios plugin to check the length of the FTS update log. """ import sys + sys.path.append('/home/zulip/deployments/current') try: from scripts.lib.setup_path import setup_path diff --git a/puppet/zulip/files/postgresql/pg_backup_and_purge b/puppet/zulip/files/postgresql/pg_backup_and_purge index e4cd6d3d39..96c1fc5979 100755 --- a/puppet/zulip/files/postgresql/pg_backup_and_purge +++ b/puppet/zulip/files/postgresql/pg_backup_and_purge @@ -1,16 +1,16 @@ #!/usr/bin/env python3 - -import os -import sys import glob +import logging +import os import shlex import subprocess -import logging -import dateutil.parser +import sys import time from datetime import datetime, timedelta, timezone from typing import Dict, List +import dateutil.parser + logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s") logger = logging.getLogger(__name__) diff --git a/puppet/zulip/files/postgresql/process_fts_updates b/puppet/zulip/files/postgresql/process_fts_updates index 8aedfe20ff..d79a6bcc1e 100755 --- a/puppet/zulip/files/postgresql/process_fts_updates +++ b/puppet/zulip/files/postgresql/process_fts_updates @@ -23,14 +23,15 @@ except ImportError: pass import argparse +import configparser +import logging +import os +import select +import sys +import time + import psycopg2 import psycopg2.extensions -import select -import time -import logging -import configparser -import sys -import os BATCH_SIZE = 1000 diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors index 7e66611cbf..fe498f3e04 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_personal_zephyr_mirrors @@ -8,9 +8,9 @@ This script works by just monitoring the files under mirrors when they receive the messages sent every minute by /etc/cron.d/test_zephyr_personal_mirrors """ -from typing import Dict import os import time +from typing import Dict RESULTS_DIR: str = "/home/zulip/mirror_status" diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness index 13b1fdff8a..59c4cd4303 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness @@ -24,9 +24,9 @@ sys.path.append('/home/zulip/deployments/current/zerver') django.setup() -from zerver.models import UserActivity +from typing import Any, Dict, Optional, Set -from typing import Any, Dict, Set, Optional +from zerver.models import UserActivity states: Dict[str, int] = { "OK": 0, diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror index ed7d8a9ed5..d095b257ad 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_zephyr_mirror @@ -9,9 +9,9 @@ run out of cron. See puppet/zulip_ops/files/cron.d/zephyr-mirror for the crontab details. """ -from typing import Dict import os import time +from typing import Dict RESULTS_FILE = "/var/lib/nagios_state/check-mirroring-results" diff --git a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces index d5ecbcf852..9c94333e94 100755 --- a/puppet/zulip_ops/files/zulip-ec2-configure-interfaces +++ b/puppet/zulip_ops/files/zulip-ec2-configure-interfaces @@ -40,16 +40,15 @@ Note that it currently does not handle the deconfiguration of interfaces. ''' - -import sys import logging import logging.handlers import subprocess +import sys +from typing import Optional import boto.utils import netifaces -from typing import Optional def address_of(device_id: int) -> Optional[str]: try: diff --git a/scripts/lib/check_rabbitmq_queue.py b/scripts/lib/check_rabbitmq_queue.py index cb4095539d..614728aa8e 100644 --- a/scripts/lib/check_rabbitmq_queue.py +++ b/scripts/lib/check_rabbitmq_queue.py @@ -1,9 +1,8 @@ +import json import os import re -import time import subprocess -import json - +import time from collections import defaultdict from typing import Any, DefaultDict, Dict, List diff --git a/scripts/lib/clean-unused-caches b/scripts/lib/clean-unused-caches index 480c6a2e03..5c548918bc 100755 --- a/scripts/lib/clean-unused-caches +++ b/scripts/lib/clean-unused-caches @@ -4,8 +4,9 @@ import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) +from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache from scripts.lib.zulip_tools import parse_cache_script_args -from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + def main() -> None: args = parse_cache_script_args("This script cleans unused zulip caches.") diff --git a/scripts/lib/clean_emoji_cache.py b/scripts/lib/clean_emoji_cache.py index aaa6936437..6457949c54 100755 --- a/scripts/lib/clean_emoji_cache.py +++ b/scripts/lib/clean_emoji_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, \ - parse_cache_script_args, purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() EMOJI_CACHE_PATH = "/srv/zulip-emoji-cache" diff --git a/scripts/lib/clean_node_cache.py b/scripts/lib/clean_node_cache.py index f4b2868738..b1fd1db0c3 100755 --- a/scripts/lib/clean_node_cache.py +++ b/scripts/lib/clean_node_cache.py @@ -2,14 +2,16 @@ import argparse import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() NODE_MODULES_CACHE_PATH = "/srv/zulip-npm-cache" diff --git a/scripts/lib/clean_venv_cache.py b/scripts/lib/clean_venv_cache.py index dd03b8aa99..513b4d532f 100755 --- a/scripts/lib/clean_venv_cache.py +++ b/scripts/lib/clean_venv_cache.py @@ -3,15 +3,17 @@ import argparse import glob import os import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) from scripts.lib.hash_reqs import expand_reqs, hash_deps -from scripts.lib.zulip_tools import \ - get_environment, get_recent_deployments, parse_cache_script_args, \ - purge_unused_caches +from scripts.lib.zulip_tools import ( + get_environment, + get_recent_deployments, + parse_cache_script_args, + purge_unused_caches, +) ENV = get_environment() VENV_CACHE_DIR = '/srv/zulip-venv-cache' diff --git a/scripts/lib/create-production-venv b/scripts/lib/create-production-venv index d7b50cd76b..16c27a4a13 100755 --- a/scripts/lib/create-production-venv +++ b/scripts/lib/create-production-venv @@ -1,17 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, overwrite_symlink, run, parse_os_release -from scripts.lib.setup_venv import ( - setup_virtualenv, get_venv_dependencies, -) +from scripts.lib.setup_venv import get_venv_dependencies, setup_virtualenv +from scripts.lib.zulip_tools import os_families, overwrite_symlink, parse_os_release, run parser = argparse.ArgumentParser(description="Create a production virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/create-thumbor-venv b/scripts/lib/create-thumbor-venv index 3ecf1bf533..f22a8c7219 100755 --- a/scripts/lib/create-thumbor-venv +++ b/scripts/lib/create-thumbor-venv @@ -1,17 +1,18 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import os_families, run, parse_os_release from scripts.lib.setup_venv import ( - setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, + YUM_THUMBOR_VENV_DEPENDENCIES, + setup_virtualenv, ) +from scripts.lib.zulip_tools import os_families, parse_os_release, run parser = argparse.ArgumentParser(description="Create a thumbor virtualenv with caching") parser.add_argument("deploy_path") diff --git a/scripts/lib/email-mirror-postfix b/scripts/lib/email-mirror-postfix index e2cd3d58e0..7b793cb7ae 100755 --- a/scripts/lib/email-mirror-postfix +++ b/scripts/lib/email-mirror-postfix @@ -40,21 +40,16 @@ Also you can use optional keys to configure the script and change default values -t Disable sending request to the Zulip server. Default value: False. """ - +import argparse +import json import os +import posix import ssl import sys - -import argparse - -import posix -import json - -from urllib.parse import urljoin, urlencode -from urllib.request import Request, urlopen -from urllib.error import HTTPError from configparser import RawConfigParser - +from urllib.error import HTTPError +from urllib.parse import urlencode, urljoin +from urllib.request import Request, urlopen parser = argparse.ArgumentParser() diff --git a/scripts/lib/hash_reqs.py b/scripts/lib/hash_reqs.py index 6115b5ef12..219ee20445 100755 --- a/scripts/lib/hash_reqs.py +++ b/scripts/lib/hash_reqs.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -import os -import sys import argparse import hashlib +import os +import sys from typing import Iterable, List, MutableSet + def expand_reqs_helper(fpath: str, visited: MutableSet[str]) -> List[str]: if fpath in visited: return [] diff --git a/scripts/lib/node_cache.py b/scripts/lib/node_cache.py index 42181daab2..b292e8c3af 100644 --- a/scripts/lib/node_cache.py +++ b/scripts/lib/node_cache.py @@ -1,10 +1,10 @@ -import os import hashlib import json +import os import shutil +from typing import List, Optional -from typing import Optional, List -from scripts.lib.zulip_tools import subprocess_text_output, run +from scripts.lib.zulip_tools import run, subprocess_text_output ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_SRV_PATH = "/srv" diff --git a/scripts/lib/pythonrc.py b/scripts/lib/pythonrc.py index 2b5e4a9a16..016d180943 100644 --- a/scripts/lib/pythonrc.py +++ b/scripts/lib/pythonrc.py @@ -1,8 +1,9 @@ try: from django.conf import settings # noqa: F401 - from zerver.models import * # noqa: F401, F403 - from zerver.lib.actions import * # noqa: F401, F403 + from analytics.models import * # noqa: F401, F403 + from zerver.lib.actions import * # noqa: F401, F403 + from zerver.models import * # noqa: F401, F403 except Exception: import traceback print("\nException importing Zulip core modules on startup!") diff --git a/scripts/lib/queue_workers.py b/scripts/lib/queue_workers.py index 12de371992..68b337c38a 100755 --- a/scripts/lib/queue_workers.py +++ b/scripts/lib/queue_workers.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import sys @@ -13,6 +12,7 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() from zerver.worker.queue_processors import get_active_worker_queues diff --git a/scripts/lib/setup_path.py b/scripts/lib/setup_path.py index c8748bd9da..20a190242a 100644 --- a/scripts/lib/setup_path.py +++ b/scripts/lib/setup_path.py @@ -1,10 +1,10 @@ """ Use libraries from a virtualenv (by modifying sys.path) in production. """ - import os import sys + def setup_path() -> None: if os.path.basename(sys.prefix) != "zulip-py3-venv": BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index b3c696806f..155df9818e 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -2,10 +2,10 @@ import logging import os import shutil import subprocess -from scripts.lib.zulip_tools import run, run_as_root, ENDC, WARNING, os_families -from scripts.lib.hash_reqs import expand_reqs +from typing import List, Optional, Set, Tuple -from typing import List, Optional, Tuple, Set +from scripts.lib.hash_reqs import expand_reqs +from scripts.lib.zulip_tools import ENDC, WARNING, os_families, run, run_as_root ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) VENV_CACHE_PATH = "/srv/zulip-venv-cache" diff --git a/scripts/lib/sharding.py b/scripts/lib/sharding.py index df68fbfe09..fad230cca0 100755 --- a/scripts/lib/sharding.py +++ b/scripts/lib/sharding.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import json import os import subprocess @@ -14,6 +13,7 @@ setup_path() from scripts.lib.zulip_tools import get_config_file + def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None: f.write("""if ($host = '{}') {{ set $tornado_server http://tornado{}; diff --git a/scripts/lib/unpack-zulip b/scripts/lib/unpack-zulip index 4d73e1b257..bf22d02053 100755 --- a/scripts/lib/unpack-zulip +++ b/scripts/lib/unpack-zulip @@ -1,17 +1,24 @@ #!/usr/bin/env python3 +import glob import os import shutil -import sys import subprocess +import sys import tempfile -import glob os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', "..")) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, make_deploy_path, \ - get_deployment_version, is_invalid_upgrade, overwrite_symlink import version +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + get_deployment_version, + is_invalid_upgrade, + make_deploy_path, + overwrite_symlink, +) if len(sys.argv) != 2: print(FAIL + f"Usage: {sys.argv[0]} " + ENDC) diff --git a/scripts/lib/upgrade-zulip b/scripts/lib/upgrade-zulip index 1454b4a0b0..5a1105ea19 100755 --- a/scripts/lib/upgrade-zulip +++ b/scripts/lib/upgrade-zulip @@ -1,19 +1,27 @@ #!/usr/bin/env python3 +import configparser +import logging import os import shutil -import sys import subprocess -import logging +import sys import time -import configparser TARBALL_ARCHIVE_PATH = "/home/zulip/archives" os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, FAIL, ENDC, \ - su_to_zulip, get_deployment_lock, release_deployment_lock, assert_running_as_root, \ - get_config_file, get_deploy_options +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + ENDC, + FAIL, + assert_running_as_root, + get_config_file, + get_deploy_options, + get_deployment_lock, + release_deployment_lock, + su_to_zulip, +) config_file: configparser.RawConfigParser = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-from-git b/scripts/lib/upgrade-zulip-from-git index 406354b14b..2359bf9f2c 100755 --- a/scripts/lib/upgrade-zulip-from-git +++ b/scripts/lib/upgrade-zulip-from-git @@ -1,18 +1,27 @@ #!/usr/bin/env python3 -import os -import sys -import subprocess -import logging -import time import argparse +import logging +import os +import subprocess +import sys +import time LOCAL_GIT_CACHE_DIR = '/srv/zulip.git' os.environ["PYTHONUNBUFFERED"] = "y" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, make_deploy_path, \ - get_deployment_lock, overwrite_symlink, release_deployment_lock, su_to_zulip, assert_running_as_root, \ - get_config_file, get_deploy_options, get_config +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + get_config, + get_config_file, + get_deploy_options, + get_deployment_lock, + make_deploy_path, + overwrite_symlink, + release_deployment_lock, + su_to_zulip, +) config_file = get_config_file() deploy_options = get_deploy_options(config_file) diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index 8cbc48d730..3677f3a61c 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -8,10 +8,10 @@ import argparse import configparser import glob import hashlib -import subprocess -import os -import sys import logging +import os +import subprocess +import sys import time os.environ["PYTHONUNBUFFERED"] = "y" @@ -22,8 +22,12 @@ os.environ["LANG"] = "en_US.UTF-8" os.environ["LANGUAGE"] = "en_US.UTF-8" sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, su_to_zulip, \ - assert_running_as_root, parse_os_release +from scripts.lib.zulip_tools import ( + DEPLOYMENTS_DIR, + assert_running_as_root, + parse_os_release, + su_to_zulip, +) assert_running_as_root() diff --git a/scripts/lib/zulip_tools.py b/scripts/lib/zulip_tools.py index dce756d64c..39e96956f7 100755 --- a/scripts/lib/zulip_tools.py +++ b/scripts/lib/zulip_tools.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 import argparse +import configparser import datetime import functools import hashlib +import json import logging import os import pwd @@ -13,11 +15,8 @@ import subprocess import sys import tempfile import time -import json import uuid -import configparser - -from typing import Sequence, Set, Any, Dict, List +from typing import Any, Dict, List, Sequence, Set DEPLOYMENTS_DIR = "/home/zulip/deployments" LOCK_DIR = os.path.join(DEPLOYMENTS_DIR, "lock") diff --git a/scripts/nagios/check-rabbitmq-consumers b/scripts/nagios/check-rabbitmq-consumers index cf4bf7e0ca..8cf11a9f2f 100755 --- a/scripts/nagios/check-rabbitmq-consumers +++ b/scripts/nagios/check-rabbitmq-consumers @@ -1,12 +1,11 @@ #!/usr/bin/env python3 - -import sys -import time import argparse import configparser -from collections import defaultdict import os import subprocess +import sys +import time +from collections import defaultdict from typing import Dict ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/scripts/nagios/cron_file_helper.py b/scripts/nagios/cron_file_helper.py index f8a7954e3c..ca92ece168 100644 --- a/scripts/nagios/cron_file_helper.py +++ b/scripts/nagios/cron_file_helper.py @@ -1,7 +1,7 @@ import time - from typing import Tuple + def nagios_from_file(results_file: str) -> Tuple[int, str]: """Returns a nagios-appropriate string and return code obtained by parsing the desired file on disk. The file on disk should be of format diff --git a/scripts/purge-old-deployments b/scripts/purge-old-deployments index f82759c94b..a1fcc56df7 100755 --- a/scripts/purge-old-deployments +++ b/scripts/purge-old-deployments @@ -3,13 +3,12 @@ import argparse import os import subprocess import sys - from typing import Set ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, \ - may_be_perform_purging +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, get_recent_deployments, may_be_perform_purging + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( diff --git a/scripts/restart-server b/scripts/restart-server index 6a0bb789e5..33b2f132c5 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -1,16 +1,16 @@ #!/usr/bin/env python3 import argparse import configparser -import os -import sys -import pwd -import subprocess import logging -import time +import os +import pwd import shlex +import subprocess +import sys +import time sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import ENDC, OKGREEN, WARNING, DEPLOYMENTS_DIR, overwrite_symlink +from scripts.lib.zulip_tools import DEPLOYMENTS_DIR, ENDC, OKGREEN, WARNING, overwrite_symlink logging.Formatter.converter = time.gmtime logging.basicConfig(format="%(asctime)s restart-server: %(message)s", diff --git a/scripts/setup/flush-memcached b/scripts/setup/flush-memcached index d9c59ddd4c..dafc37bc6b 100755 --- a/scripts/setup/flush-memcached +++ b/scripts/setup/flush-memcached @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -10,9 +9,10 @@ from scripts.lib.setup_path import setup_path setup_path() -from zproject import settings import pylibmc +from zproject import settings + pylibmc.Client( [settings.MEMCACHED_LOCATION], binary=True, diff --git a/scripts/setup/generate_secrets.py b/scripts/setup/generate_secrets.py index 864c684ec7..1bb880e278 100755 --- a/scripts/setup/generate_secrets.py +++ b/scripts/setup/generate_secrets.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # This tools generates /etc/zulip/zulip-secrets.conf - -import sys import os - +import sys from typing import Dict, List BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -15,8 +13,8 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import argparse -import uuid import configparser +import uuid os.chdir(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -140,6 +138,7 @@ def generate_secrets(development: bool = False) -> None: # file directly. import redis + from zerver.lib.redis_utils import get_redis_client redis_password = random_token() diff --git a/scripts/setup/inline_email_css.py b/scripts/setup/inline_email_css.py index 58bf6bdc3e..418ff150f4 100755 --- a/scripts/setup/inline_email_css.py +++ b/scripts/setup/inline_email_css.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import os - -from premailer import Premailer -from cssutils import profile -from cssutils.profiles import Profiles, properties, macros from typing import Set +from cssutils import profile +from cssutils.profiles import Profiles, macros, properties +from premailer import Premailer + ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') EMAIL_TEMPLATES_PATH = os.path.join(ZULIP_PATH, 'templates', 'zerver', 'emails') COMPILED_EMAIL_TEMPLATES_PATH = os.path.join(EMAIL_TEMPLATES_PATH, 'compiled') diff --git a/scripts/setup/restore-backup b/scripts/setup/restore-backup index 00f3b3e713..432cf4a61c 100755 --- a/scripts/setup/restore-backup +++ b/scripts/setup/restore-backup @@ -1,17 +1,15 @@ #!/usr/bin/env python3 - import argparse import os import re import subprocess import sys import tempfile - from typing import IO BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(BASE_DIR) -from scripts.lib.zulip_tools import su_to_zulip, run +from scripts.lib.zulip_tools import run, su_to_zulip POSTGRES_USER = "postgres" diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 672591c44a..c0e1194c70 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -1,11 +1,11 @@ #!/usr/bin/env python3 - -import os -import sys -import subprocess import configparser +import os import re -from lib.zulip_tools import parse_os_release, assert_running_as_root +import subprocess +import sys + +from lib.zulip_tools import assert_running_as_root, parse_os_release assert_running_as_root() BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/static/assets/favicon/generate b/static/assets/favicon/generate index 9ba478fd64..ae94f76001 100755 --- a/static/assets/favicon/generate +++ b/static/assets/favicon/generate @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import xml.etree.ElementTree as ET import subprocess +from xml.etree import ElementTree as ET # Generates the favicon images containing unread message counts. diff --git a/tools/check-capitalization b/tools/check-capitalization index c99f08ceab..1645ed9816 100755 --- a/tools/check-capitalization +++ b/tools/check-capitalization @@ -2,6 +2,7 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse @@ -12,8 +13,7 @@ import subprocess import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC - +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING from tools.lib.capitalization import check_capitalization DJANGO_PO_REGEX = re.compile('msgid "(.*?)"') diff --git a/tools/check-frontend-i18n b/tools/check-frontend-i18n index 24c40a70da..6c119cd201 100755 --- a/tools/check-frontend-i18n +++ b/tools/check-frontend-i18n @@ -1,18 +1,20 @@ #!/usr/bin/env python3 - from typing import List + # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) import argparse import json import os -import sys import subprocess +import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import ENDC, FAIL, WARNING + def find_handlebars(translatable_strings: List[str]) -> List[str]: errored = [] diff --git a/tools/check-issue-labels b/tools/check-issue-labels index fea01f1fc7..5115df2fe6 100755 --- a/tools/check-issue-labels +++ b/tools/check-issue-labels @@ -1,13 +1,13 @@ #!/usr/bin/env python3 +import argparse +import os +import re +import sys +from typing import Any, Dict, Optional import requests -import re -import argparse -import sys -import os import ConfigParser -from typing import Any, Dict, Optional # Scans zulip repositary for issues that don't have any `area` labels. # GitHub API token is required as GitHub limits unauthenticated diff --git a/tools/check-provision b/tools/check-provision index 41d5a5bcbb..fe64e18daa 100755 --- a/tools/check-provision +++ b/tools/check-provision @@ -1,16 +1,14 @@ #!/usr/bin/env python3 - -import os import argparse +import os import sys tools_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) +from tools.lib.test_script import assert_provisioning_status_ok + def run() -> None: parser = argparse.ArgumentParser() diff --git a/tools/check-templates b/tools/check-templates index 7d1859cbb4..deb796482a 100755 --- a/tools/check-templates +++ b/tools/check-templates @@ -1,18 +1,20 @@ #!/usr/bin/env python3 -from lib.template_parser import validate -from lib.html_branches import build_id_dict -from lib.pretty_print import validate_indent_html import argparse -import sys import logging +import sys # check for the venv from lib import sanity_check +from lib.html_branches import build_id_dict +from lib.pretty_print import validate_indent_html +from lib.template_parser import validate + sanity_check.check_venv(__file__) -from zulint import lister from typing import Dict, Iterable, List +from zulint import lister + EXCLUDED_FILES = [ ## Test data Files for testing modules in tests "tools/tests/test_template_data", diff --git a/tools/check-thirdparty b/tools/check-thirdparty index 501b8f5c98..fc0f0b0175 100755 --- a/tools/check-thirdparty +++ b/tools/check-thirdparty @@ -7,12 +7,12 @@ Disclaimer: This script is not a lawyer. It cannot validate that the claimed licenses are correct. It can only check for basic syntactic issues. """ - import difflib import io import os import subprocess import sys + from debian import copyright COPYRIGHT_FILENAME = "docs/THIRDPARTY" diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 2c313659d2..7ba2beb421 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -2,14 +2,15 @@ # check for the venv from lib import sanity_check -sanity_check.check_venv(__file__) -from collections import defaultdict -from typing import Any, Dict, List, Set +sanity_check.check_venv(__file__) import html import os import pprint +from collections import defaultdict +from typing import Any, Dict, List, Set + import ujson Call = Dict[str, Any] diff --git a/tools/diagnose b/tools/diagnose index 657ab87520..80f1ce75e7 100755 --- a/tools/diagnose +++ b/tools/diagnose @@ -1,17 +1,16 @@ #!/usr/bin/env python3 - import os import platform import shlex -import sys import subprocess - +import sys from typing import Callable, List TOOLS_DIR = os.path.dirname(__file__) ROOT_DIR = os.path.dirname(TOOLS_DIR) sys.path.insert(0, ROOT_DIR) from scripts.lib.zulip_tools import get_dev_uuid_var_path + UUID_VAR_PATH = get_dev_uuid_var_path() def run(check_func: Callable[[], bool]) -> None: @@ -88,7 +87,7 @@ def test_models() -> bool: os.environ['DJANGO_SETTINGS_MODULE'] = settings_module import django django.setup() - from zerver.models import UserProfile, Realm + from zerver.models import Realm, UserProfile print('Num realms: ', Realm.objects.count()) print('Num users: ', UserProfile.objects.count()) return True diff --git a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py index 7f51832fc1..010af602b1 100644 --- a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py +++ b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py @@ -1,8 +1,9 @@ import optparse -from scrapy.crawler import Crawler -from scrapy.commands import crawl from typing import List, Union +from scrapy.commands import crawl +from scrapy.crawler import Crawler + class Command(crawl.Command): def run(self, args: List[str], opts: optparse.Values) -> None: diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py index 861553dad4..86dce36313 100755 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py @@ -1,6 +1,5 @@ import os import pathlib - from typing import List from .common.spiders import BaseDocumentationSpider diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py index 4b6bb43ac8..082001eaee 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_help_documentation.py @@ -1,12 +1,10 @@ import os - from posixpath import basename +from typing import Any, List, Set from urllib.parse import urlparse from .common.spiders import BaseDocumentationSpider -from typing import Any, List, Set - def get_images_dir(images_path: str) -> str: # Get index html file as start url and convert it to file uri diff --git a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py index b75cdf8570..6d17c789a5 100644 --- a/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py @@ -1,7 +1,8 @@ import json import re -import scrapy +from typing import Callable, Iterable, List, Optional, Union +import scrapy from scrapy.http import Request, Response from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor @@ -9,8 +10,6 @@ from scrapy.spidermiddlewares.httperror import HttpError from scrapy.utils.url import url_has_any_extension from twisted.python.failure import Failure -from typing import Callable, Iterable, List, Optional, Union - EXCLUDED_URLS = [ # Google calendar returns 404s on HEAD requests unconditionally 'https://calendar.google.com/calendar/embed?src=ktiduof4eoh47lmgcl2qunnc0o@group.calendar.google.com', diff --git a/tools/droplets/add_mentor.py b/tools/droplets/add_mentor.py index f505828120..75a977f973 100644 --- a/tools/droplets/add_mentor.py +++ b/tools/droplets/add_mentor.py @@ -9,13 +9,12 @@ # machine: # # $ python3 add_mentor.py --remove - import os +import re +import socket import sys from argparse import ArgumentParser from typing import List -import socket -import re import requests diff --git a/tools/droplets/create.py b/tools/droplets/create.py index 0c2b8e6e2a..320f01fbb4 100644 --- a/tools/droplets/create.py +++ b/tools/droplets/create.py @@ -14,19 +14,18 @@ # Copy conf.ini-template to conf.ini and populate with your api token. # # usage: python3 create.py - -import sys +import argparse import configparser +import json +import os +import sys +import time import urllib.error import urllib.request -import json -import digitalocean -import time -import argparse -import os - from typing import Any, Dict, List +import digitalocean + # initiation argument parser parser = argparse.ArgumentParser(description='Create a Zulip devopment VM Digital Ocean droplet.') parser.add_argument("username", help="Github username for whom you want to create a Zulip dev droplet") diff --git a/tools/fetch-contributor-data b/tools/fetch-contributor-data index 40a82d06fd..327b921e44 100755 --- a/tools/fetch-contributor-data +++ b/tools/fetch-contributor-data @@ -3,31 +3,32 @@ Fetch contributors data from Github using their API, convert it to structured JSON data for the /team page contributors section. """ - import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from scripts.lib.setup_path import setup_path + setup_path() -from typing import Any, Dict, List, Optional, Union -from typing_extensions import TypedDict - import argparse -from time import sleep +import logging from datetime import date from random import randrange -import logging +from time import sleep +from typing import Any, Dict, List, Optional, Union + +from typing_extensions import TypedDict os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' import django + django.setup() -from django.conf import settings +import json import requests -import json +from django.conf import settings duplicate_commits_file = os.path.join(os.path.dirname(__file__), 'duplicate_commits.json') diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot index c7b419e511..44aefa6fd8 100755 --- a/tools/generate-integration-docs-screenshot +++ b/tools/generate-integration-docs-screenshot @@ -5,7 +5,6 @@ from lib import sanity_check sanity_check.check_venv(__file__) - import os import sys @@ -26,23 +25,33 @@ import argparse import base64 import subprocess from typing import Any, Dict, Optional -from urllib.parse import urlencode, parse_qsl +from urllib.parse import parse_qsl, urlencode import requests import ujson from scripts.lib.zulip_tools import BOLDRED, ENDC from tools.lib.test_script import prepare_puppeteer_run -from zerver.models import UserProfile, Message, get_user_by_delivery_email, get_realm from zerver.lib.actions import ( - do_create_user, notify_created_bot, bulk_add_subscriptions, do_change_avatar_fields) + bulk_add_subscriptions, + do_change_avatar_fields, + do_create_user, + notify_created_bot, +) +from zerver.lib.integrations import ( + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.storage import static_path from zerver.lib.streams import create_stream_if_needed from zerver.lib.upload import upload_avatar_image -from zerver.lib.integrations import ( - WebhookIntegration, INTEGRATIONS, split_fixture_path, ScreenshotConfig, get_fixture_and_image_paths, - DOC_SCREENSHOT_CONFIG) from zerver.lib.webhooks.common import get_fixture_http_headers -from zerver.lib.storage import static_path +from zerver.models import Message, UserProfile, get_realm, get_user_by_delivery_email + def create_integration_bot(integration: WebhookIntegration, bot_name: Optional[str]=None) -> UserProfile: realm = get_realm('zulip') diff --git a/tools/i18n/process-mobile-i18n b/tools/i18n/process-mobile-i18n index e8be13254d..1ce108879e 100755 --- a/tools/i18n/process-mobile-i18n +++ b/tools/i18n/process-mobile-i18n @@ -5,6 +5,7 @@ import re from subprocess import check_output from typing import Dict, List + def get_json_filename(locale: str) -> str: return f"locale/{locale}/mobile.json" diff --git a/tools/i18n/tagmessages b/tools/i18n/tagmessages index 853a4d2044..9ad637d8b1 100755 --- a/tools/i18n/tagmessages +++ b/tools/i18n/tagmessages @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import configparser -from hashlib import md5 import os +from hashlib import md5 import polib diff --git a/tools/lib/capitalization.py b/tools/lib/capitalization.py index 49aaba7db8..3386d47cfc 100644 --- a/tools/lib/capitalization.py +++ b/tools/lib/capitalization.py @@ -1,5 +1,5 @@ -from typing import List, Tuple, Match import re +from typing import List, Match, Tuple from bs4 import BeautifulSoup diff --git a/tools/lib/gitlint-rules.py b/tools/lib/gitlint-rules.py index 810267dd44..de77c52cc5 100644 --- a/tools/lib/gitlint-rules.py +++ b/tools/lib/gitlint-rules.py @@ -1,9 +1,9 @@ -from typing import Text, List +import re +from typing import List, Text from gitlint.git import GitCommit -from gitlint.rules import LineRule, RuleViolation, CommitMessageTitle from gitlint.options import StrOption -import re +from gitlint.rules import CommitMessageTitle, LineRule, RuleViolation # Word list from https://github.com/m1foley/fit-commit # Copyright (c) 2015 Mike Foley diff --git a/tools/lib/html_branches.py b/tools/lib/html_branches.py index fbfce1a2ec..ceb588e46d 100644 --- a/tools/lib/html_branches.py +++ b/tools/lib/html_branches.py @@ -1,13 +1,8 @@ -from typing import Dict, List, Optional, Set - import re from collections import defaultdict +from typing import Dict, List, Optional, Set -from .template_parser import ( - tokenize, - FormattedException, - Token, -) +from .template_parser import FormattedException, Token, tokenize class HtmlBranchesException(Exception): diff --git a/tools/lib/pretty_print.py b/tools/lib/pretty_print.py index 6f72984f4b..2abc94ac2b 100644 --- a/tools/lib/pretty_print.py +++ b/tools/lib/pretty_print.py @@ -1,13 +1,10 @@ +import subprocess from typing import Any, Dict, List -from .template_parser import ( - tokenize, - is_django_block_tag, -) +from zulint.printer import ENDC, GREEN -from zulint.printer import GREEN, ENDC +from .template_parser import is_django_block_tag, tokenize -import subprocess def pretty_print_html(html: str, num_spaces: int = 4) -> str: # We use 1-based indexing for both rows and columns. diff --git a/tools/lib/provision.py b/tools/lib/provision.py index 7d949ba5ae..b52b94d524 100755 --- a/tools/lib/provision.py +++ b/tools/lib/provision.py @@ -1,28 +1,37 @@ #!/usr/bin/env python3 -import os -import sys -import logging import argparse +import hashlib +import logging +import os import platform import subprocess -import hashlib +import sys os.environ["PYTHONUNBUFFERED"] = "y" ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \ - get_dev_uuid_var_path, FAIL, os_families, parse_os_release, \ - overwrite_symlink +from typing import TYPE_CHECKING, List + +from scripts.lib.node_cache import NODE_MODULES_CACHE_PATH, setup_node_modules from scripts.lib.setup_venv import ( - get_venv_dependencies, THUMBOR_VENV_DEPENDENCIES, + THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES, + get_venv_dependencies, +) +from scripts.lib.zulip_tools import ( + ENDC, + FAIL, + WARNING, + get_dev_uuid_var_path, + os_families, + overwrite_symlink, + parse_os_release, + run_as_root, ) -from scripts.lib.node_cache import setup_node_modules, NODE_MODULES_CACHE_PATH from tools.setup import setup_venvs -from typing import List, TYPE_CHECKING if TYPE_CHECKING: from typing import NoReturn diff --git a/tools/lib/provision_inner.py b/tools/lib/provision_inner.py index 3f09e9ed8e..6582e2d2d8 100755 --- a/tools/lib/provision_inner.py +++ b/tools/lib/provision_inner.py @@ -1,22 +1,26 @@ #!/usr/bin/env python3 -import os -import sys import argparse import glob +import os import shutil - +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from scripts.lib.zulip_tools import run, OKBLUE, ENDC, \ - get_dev_uuid_var_path, is_digest_obsolete, write_new_digest - -from version import PROVISION_VERSION from pygments import __version__ as pygments_version +from scripts.lib.zulip_tools import ( + ENDC, + OKBLUE, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, +) from tools.setup.generate_zulip_bots_static_files import generate_zulip_bots_static_files +from version import PROVISION_VERSION VENV_PATH = "/srv/zulip-py3-venv" UUID_VAR_PATH = get_dev_uuid_var_path() @@ -177,7 +181,7 @@ def clean_unused_caches() -> None: verbose=False, no_headings=True, ) - from scripts.lib import clean_venv_cache, clean_node_cache, clean_emoji_cache + from scripts.lib import clean_emoji_cache, clean_node_cache, clean_venv_cache clean_venv_cache.main(args) clean_node_cache.main(args) clean_emoji_cache.main(args) @@ -227,12 +231,13 @@ def main(options: argparse.Namespace) -> int: import django django.setup() + from django.conf import settings + from zerver.lib.test_fixtures import ( DEV_DATABASE, TEST_DATABASE, destroy_leaked_test_databases, ) - from django.conf import settings if options.is_force or need_to_run_configure_rabbitmq( [settings.RABBITMQ_PASSWORD]): diff --git a/tools/lib/sanity_check.py b/tools/lib/sanity_check.py index dac8209fdb..6ebe777b90 100644 --- a/tools/lib/sanity_check.py +++ b/tools/lib/sanity_check.py @@ -2,6 +2,7 @@ import os import pwd import sys + def check_venv(filename: str) -> None: try: import django diff --git a/tools/lib/template_parser.py b/tools/lib/template_parser.py index 124f500dfd..b53bde07ba 100644 --- a/tools/lib/template_parser.py +++ b/tools/lib/template_parser.py @@ -1,5 +1,6 @@ from typing import Callable, List, Optional, Text + class FormattedException(Exception): pass diff --git a/tools/lib/test_script.py b/tools/lib/test_script.py index ca03ffae09..5c7780262b 100644 --- a/tools/lib/test_script.py +++ b/tools/lib/test_script.py @@ -1,12 +1,12 @@ -from typing import Optional, Tuple, Iterable, List - +import glob import os import subprocess import sys from distutils.version import LooseVersion -from version import PROVISION_VERSION +from typing import Iterable, List, Optional, Tuple + from scripts.lib.zulip_tools import get_dev_uuid_var_path -import glob +from version import PROVISION_VERSION ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tools/lib/test_server.py b/tools/lib/test_server.py index 5a1a66429e..5e47a8f343 100644 --- a/tools/lib/test_server.py +++ b/tools/lib/test_server.py @@ -2,13 +2,12 @@ import os import subprocess import sys import time - from contextlib import contextmanager - from typing import Iterator, Optional # Verify the Zulip venv is available. from tools.lib import sanity_check + sanity_check.check_venv(__file__) import django @@ -20,8 +19,9 @@ TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if TOOLS_DIR not in sys.path: sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from zerver.lib.test_fixtures import update_test_databases_if_required from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.test_fixtures import update_test_databases_if_required + def set_up_django(external_host: str) -> None: os.environ['EXTERNAL_HOST'] = external_host diff --git a/tools/lint b/tools/lint index e222cb6c08..2c379c83e5 100755 --- a/tools/lint +++ b/tools/lint @@ -1,16 +1,20 @@ #!/usr/bin/env python3 +import argparse import os import sys -import argparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from linter_lib.custom_check import python_rules, non_py_rules -from zulint.command import add_default_linter_arguments, LinterConfig import random +from zulint.command import LinterConfig, add_default_linter_arguments + +from linter_lib.custom_check import non_py_rules, python_rules + + def run() -> None: parser = argparse.ArgumentParser() parser.add_argument('--force', default=False, @@ -26,13 +30,10 @@ def run() -> None: root_dir = os.path.dirname(tools_dir) sys.path.insert(0, root_dir) + from tools.lib.test_script import assert_provisioning_status_ok from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE - from tools.linter_lib.pyflakes import check_pyflakes from tools.linter_lib.pep8 import check_pep8 - - from tools.lib.test_script import ( - assert_provisioning_status_ok, - ) + from tools.linter_lib.pyflakes import check_pyflakes os.chdir(root_dir) diff --git a/tools/linter_lib/pep8.py b/tools/linter_lib/pep8.py index 58e330462e..93ad58de03 100644 --- a/tools/linter_lib/pep8.py +++ b/tools/linter_lib/pep8.py @@ -1,6 +1,7 @@ +from typing import List + from zulint.linters import run_pycodestyle -from typing import List def check_pep8(files: List[str]) -> bool: ignored_rules = [ diff --git a/tools/linter_lib/pyflakes.py b/tools/linter_lib/pyflakes.py index 33ecdd504f..ff630b3a38 100644 --- a/tools/linter_lib/pyflakes.py +++ b/tools/linter_lib/pyflakes.py @@ -1,5 +1,4 @@ import argparse - from typing import List from zulint.linters import run_pyflakes diff --git a/tools/pretty-print-html b/tools/pretty-print-html index d3fb88a4b3..c275c0440f 100755 --- a/tools/pretty-print-html +++ b/tools/pretty-print-html @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -from typing import List -from lib.pretty_print import pretty_print_html import sys +from typing import List + +from lib.pretty_print import pretty_print_html + def clean_html(filenames: List[str]) -> None: for fn in filenames: diff --git a/tools/renumber-migrations b/tools/renumber-migrations index 10998a8da7..600604ccbc 100755 --- a/tools/renumber-migrations +++ b/tools/renumber-migrations @@ -1,13 +1,12 @@ #!/usr/bin/env python3 - +import fileinput import glob import os -import sys -import fileinput import re - +import sys from typing import List + def validate_order(order: List[int], length: int) -> None: if len(order) != length: print("Please enter the sequence of all the conflicting files at once") diff --git a/tools/review b/tools/review index 24541b4f0b..cf2f5b7208 100755 --- a/tools/review +++ b/tools/review @@ -1,10 +1,10 @@ #!/usr/bin/env python3 - import shlex import subprocess import sys from typing import List + def exit(message: str) -> None: print('PROBLEM!') print(message) diff --git a/tools/run-dev.py b/tools/run-dev.py index 10615e874f..8ca71c13c5 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import os import pwd @@ -7,27 +6,22 @@ import signal import subprocess import sys import traceback - from urllib.parse import urlunparse # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from tornado import httpclient -from tornado import httputil -from tornado import gen -from tornado import web +from tornado import gen, httpclient, httputil, web from tornado.ioloop import IOLoop TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -from tools.lib.test_script import ( - assert_provisioning_status_ok, -) - from typing import Any, Callable, Generator, List, Optional +from tools.lib.test_script import assert_provisioning_status_ok + if 'posix' in os.name and os.geteuid() == 0: raise RuntimeError("run-dev.py should not be run as root.") @@ -103,7 +97,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = settings_module sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from scripts.lib.zulip_tools import CYAN, WARNING, FAIL, ENDC +from scripts.lib.zulip_tools import CYAN, ENDC, FAIL, WARNING proxy_port = base_port django_port = base_port + 1 diff --git a/tools/run-mypy b/tools/run-mypy index 939b667616..e72148591b 100755 --- a/tools/run-mypy +++ b/tools/run-mypy @@ -1,10 +1,8 @@ #!/usr/bin/env python3 - - -import os -import sys import argparse +import os import subprocess +import sys from typing import List from zulint import lister diff --git a/tools/setup/build_pygments_data b/tools/setup/build_pygments_data index 7c15888c35..8b55ea399c 100755 --- a/tools/setup/build_pygments_data +++ b/tools/setup/build_pygments_data @@ -1,9 +1,9 @@ #!/usr/bin/env python3 - -from pygments.lexers import get_all_lexers import json import os +from pygments.lexers import get_all_lexers + ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') DATA_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'lang.json') OUT_PATH = os.path.join(ZULIP_PATH, 'static', 'generated', 'pygments_data.json') diff --git a/tools/setup/emoji/build_emoji b/tools/setup/emoji/build_emoji index 23923323bf..1ff34775df 100755 --- a/tools/setup/emoji/build_emoji +++ b/tools/setup/emoji/build_emoji @@ -5,14 +5,20 @@ import os import shutil import sys -import ujson - from typing import Any, Dict, Iterator, List, Optional -from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \ - get_emoji_code, generate_name_to_codepoint_map, emoji_names_for_picker, \ - EMOTICON_CONVERSIONS, REMAPPED_EMOJIS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import ( + EMOTICON_CONVERSIONS, + REMAPPED_EMOJIS, + emoji_names_for_picker, + generate_codepoint_to_name_map, + generate_emoji_catalog, + generate_name_to_codepoint_map, + get_emoji_code, +) ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') sys.path.append(ZULIP_PATH) diff --git a/tools/setup/emoji/emoji_setup_utils.py b/tools/setup/emoji/emoji_setup_utils.py index 64bdfbdb9e..3daff11a0b 100644 --- a/tools/setup/emoji/emoji_setup_utils.py +++ b/tools/setup/emoji/emoji_setup_utils.py @@ -1,8 +1,6 @@ # This file contains various helper functions used by `build_emoji` tool. # See docs/subsystems/emoji.md for details on how this system works. - from collections import defaultdict - from typing import Any, Dict, List # Emojisets that we currently support. diff --git a/tools/setup/emoji/export_emoji_names_to_csv b/tools/setup/emoji/export_emoji_names_to_csv index da1ab25932..c1e44a7e96 100755 --- a/tools/setup/emoji/export_emoji_names_to_csv +++ b/tools/setup/emoji/export_emoji_names_to_csv @@ -10,12 +10,12 @@ import argparse import csv import os import re +from typing import Any, Dict, List + import ujson from emoji_setup_utils import get_emoji_code -from typing import Any, Dict, List - TOOLS_DIR_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR_PATH) # `emoji.json` file is same in all four emoji-datasource packages. diff --git a/tools/setup/emoji/generate_emoji_names_table b/tools/setup/emoji/generate_emoji_names_table index 71ecd8341e..90a5ba1e11 100755 --- a/tools/setup/emoji/generate_emoji_names_table +++ b/tools/setup/emoji/generate_emoji_names_table @@ -6,12 +6,12 @@ # sources' decisions about what names to provide to each unicode # codepoint. import os -import ujson - from typing import Any, Dict, List -from emoji_setup_utils import emoji_is_universal, get_emoji_code, EMOJISETS +import ujson + from emoji_names import EMOJI_NAME_MAPS +from emoji_setup_utils import EMOJISETS, emoji_is_universal, get_emoji_code TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/setup/emoji/import_emoji_names_from_csv b/tools/setup/emoji/import_emoji_names_from_csv index f59977f359..cd188c47b2 100755 --- a/tools/setup/emoji/import_emoji_names_from_csv +++ b/tools/setup/emoji/import_emoji_names_from_csv @@ -13,7 +13,6 @@ import csv import os import re import textwrap - from typing import Any, Dict, List, Set EMOJI_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/setup/emoji/test-emoji-name-scripts b/tools/setup/emoji/test-emoji-name-scripts index 0dce85763f..1cd7b9b1d6 100755 --- a/tools/setup/emoji/test-emoji-name-scripts +++ b/tools/setup/emoji/test-emoji-name-scripts @@ -3,8 +3,8 @@ import argparse import difflib import filecmp import os -import subprocess import shutil +import subprocess import tempfile TOOLS_DIR = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) diff --git a/tools/setup/generate_integration_bots_avatars.py b/tools/setup/generate_integration_bots_avatars.py index b5b38a1e6d..9cd2f6b8eb 100755 --- a/tools/setup/generate_integration_bots_avatars.py +++ b/tools/setup/generate_integration_bots_avatars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import sys @@ -7,6 +6,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" @@ -20,9 +20,10 @@ import io import cairosvg from PIL import Image -from zerver.lib.upload import resize_avatar, DEFAULT_AVATAR_SIZE from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.storage import static_path +from zerver.lib.upload import DEFAULT_AVATAR_SIZE, resize_avatar + def create_square_image(png: bytes) -> bytes: img = Image.open(io.BytesIO(png)) diff --git a/tools/setup/generate_zulip_bots_static_files.py b/tools/setup/generate_zulip_bots_static_files.py index 83c22fbd7b..17ba1f3d1d 100755 --- a/tools/setup/generate_zulip_bots_static_files.py +++ b/tools/setup/generate_zulip_bots_static_files.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 - import glob import os -import sys import shutil +import sys from typing import List ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if ZULIP_PATH not in sys.path: sys.path.append(ZULIP_PATH) from scripts.lib.setup_path import setup_path + setup_path() from zulip_bots.lib import get_bots_directory_path + def generate_zulip_bots_static_files() -> None: bots_dir = 'static/generated/bots' if os.path.isdir(bots_dir): diff --git a/tools/show-profile-results b/tools/show-profile-results index 0ac294cc4f..fb6aa1ce0f 100755 --- a/tools/show-profile-results +++ b/tools/show-profile-results @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import sys import pstats +import sys ''' This is a helper script to make it easy to show profile diff --git a/tools/test-api b/tools/test-api index 7685dd012e..ef61dbbc56 100755 --- a/tools/test-api +++ b/tools/test-api @@ -7,6 +7,7 @@ os.environ["RUNNING_OPENAPI_CURL_TEST"] = "1" # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -30,13 +31,13 @@ assert_provisioning_status_ok(options.force) with test_server_running(force=options.force, external_host='zulipdev.com:9981'): # Zerver imports should happen after `django.setup()` is run # by the test_server_running decorator. - from zerver.openapi.python_examples import test_the_api, test_invalid_api_key - from zerver.openapi.javascript_examples import test_js_bindings - from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success from zerver.lib.actions import do_create_user from zerver.lib.test_helpers import reset_emails_in_zulip_realm from zerver.lib.users import get_api_key - from zerver.models import get_user, get_realm + from zerver.models import get_realm, get_user + from zerver.openapi.javascript_examples import test_js_bindings + from zerver.openapi.python_examples import test_invalid_api_key, test_the_api + from zerver.openapi.test_curl_examples import test_generated_curl_examples_for_success print("Running API tests...") diff --git a/tools/test-backend b/tools/test-backend index 6f7830849d..8c8fc3b157 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -2,22 +2,23 @@ # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) -from typing import List, Iterator -import glob import argparse import contextlib -from unittest import mock +import glob import os import shlex -import sys import subprocess +import sys import tempfile -import ujson -import responses +from typing import Iterator, List +from unittest import mock import django +import responses +import ujson from django.conf import settings from django.test.utils import get_runner @@ -183,11 +184,10 @@ def main() -> None: os.environ.pop("http_proxy", "") os.environ.pop("https_proxy", "") - from zerver.lib.test_fixtures import update_test_databases_if_required, \ - remove_test_run_directories - - from tools.lib.test_script import ( - assert_provisioning_status_ok, + from tools.lib.test_script import assert_provisioning_status_ok + from zerver.lib.test_fixtures import ( + remove_test_run_directories, + update_test_databases_if_required, ) os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.test_settings' @@ -460,6 +460,7 @@ def main() -> None: if options.report_slow_tests: from zerver.lib.test_runner import report_slow_tests + # We do this even with failures, since slowness can be # an important clue as to why tests fail. report_slow_tests() diff --git a/tools/test-help-documentation b/tools/test-help-documentation index 7977c2b1c4..e0bb8aa4dc 100755 --- a/tools/test-help-documentation +++ b/tools/test-help-documentation @@ -2,14 +2,15 @@ import argparse import contextlib import os -import sys import subprocess +import sys from typing import Iterator ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/test-js-with-node b/tools/test-js-with-node index 0f9a16d55c..ca9b70c456 100755 --- a/tools/test-js-with-node +++ b/tools/test-js-with-node @@ -5,7 +5,7 @@ import os import pwd import subprocess import sys -from typing import Dict, Any +from typing import Any, Dict TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(TOOLS_DIR)) @@ -13,12 +13,12 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) # Import this after we do the sanity_check so it doesn't crash. import ujson - -from zulint.printer import CYAN, GREEN, BOLDRED, ENDC +from zulint.printer import BOLDRED, CYAN, ENDC, GREEN INDEX_JS = 'frontend_tests/zjsunit/index.js' NODE_COVERAGE_PATH = 'var/node-coverage/coverage-final.json' diff --git a/tools/test-js-with-puppeteer b/tools/test-js-with-puppeteer index fc2219b67c..9938079c46 100755 --- a/tools/test-js-with-puppeteer +++ b/tools/test-js-with-puppeteer @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import argparse -import subprocess -import sys import os import shlex +import subprocess +import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -42,12 +42,18 @@ sys.path.insert(0, ZULIP_PATH) # check for the venv from tools.lib import sanity_check + sanity_check.check_venv(__file__) -from tools.lib.test_script import assert_provisioning_status_ok, find_js_test_files, prepare_puppeteer_run +from typing import Iterable + +from tools.lib.test_script import ( + assert_provisioning_status_ok, + find_js_test_files, + prepare_puppeteer_run, +) from tools.lib.test_server import test_server_running -from typing import Iterable def run_tests(files: Iterable[str], external_host: str) -> None: test_dir = os.path.join(ZULIP_PATH, 'frontend_tests/puppeteer_tests') diff --git a/tools/test-locked-requirements b/tools/test-locked-requirements index 98cc73ee41..2390264660 100755 --- a/tools/test-locked-requirements +++ b/tools/test-locked-requirements @@ -8,9 +8,9 @@ import shutil import subprocess import sys import tempfile -import ujson +from typing import List, Optional -from typing import Optional, List +import ujson TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) ZULIP_PATH = os.path.dirname(TOOLS_DIR) diff --git a/tools/test-queue-worker-reload b/tools/test-queue-worker-reload index 152edf202e..6f64dd7d8e 100755 --- a/tools/test-queue-worker-reload +++ b/tools/test-queue-worker-reload @@ -1,15 +1,14 @@ #!/usr/bin/env python3 - - import os -import sys -import time import signal import subprocess +import sys +import time import types # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) # TODO: Convert this to use scripts/lib/queue_workers.py diff --git a/tools/test-run-dev b/tools/test-run-dev index 9835c27021..f7d882373f 100755 --- a/tools/test-run-dev +++ b/tools/test-run-dev @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import os import signal import subprocess @@ -8,6 +7,7 @@ import time from typing import Tuple from lib import sanity_check + sanity_check.check_venv(__file__) TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/test-tools b/tools/test-tools index 14775fd74c..326f726ecf 100755 --- a/tools/test-tools +++ b/tools/test-tools @@ -1,6 +1,4 @@ #!/usr/bin/env python3 - - import argparse import os import sys @@ -8,6 +6,7 @@ import unittest # check for the venv from lib import sanity_check + sanity_check.check_venv(__file__) if __name__ == '__main__': diff --git a/tools/tests/test_capitalization_checker.py b/tools/tests/test_capitalization_checker.py index 37096d7391..2789f653c7 100644 --- a/tools/tests/test_capitalization_checker.py +++ b/tools/tests/test_capitalization_checker.py @@ -1,8 +1,9 @@ -from bs4 import BeautifulSoup from unittest import TestCase -from tools.lib.capitalization import check_capitalization, is_capitalized, \ - get_safe_text +from bs4 import BeautifulSoup + +from tools.lib.capitalization import check_capitalization, get_safe_text, is_capitalized + class GetSafeTextTestCase(TestCase): def test_get_safe_text(self) -> None: diff --git a/tools/tests/test_check_rabbitmq_queue.py b/tools/tests/test_check_rabbitmq_queue.py index d135097194..2eb531e1d9 100644 --- a/tools/tests/test_check_rabbitmq_queue.py +++ b/tools/tests/test_check_rabbitmq_queue.py @@ -1,14 +1,8 @@ -from unittest import mock -from unittest import TestCase - -from scripts.lib.check_rabbitmq_queue import ( - analyze_queue_stats, - OK, - WARNING, - CRITICAL, - UNKNOWN, -) import time +from unittest import TestCase, mock + +from scripts.lib.check_rabbitmq_queue import CRITICAL, OK, UNKNOWN, WARNING, analyze_queue_stats + class AnalyzeQueueStatsTests(TestCase): def test_no_stats_available(self) -> None: diff --git a/tools/tests/test_html_branches.py b/tools/tests/test_html_branches.py index 7db3609124..36d268a255 100644 --- a/tools/tests/test_html_branches.py +++ b/tools/tests/test_html_branches.py @@ -1,13 +1,12 @@ -import unittest import os +import unittest import tools.lib.template_parser - from tools.lib.html_branches import ( + build_id_dict, get_tag_info, html_branches, html_tag_tree, - build_id_dict, split_for_id_and_class, ) diff --git a/tools/tests/test_template_parser.py b/tools/tests/test_template_parser.py index 5928101e47..421a10a1a3 100644 --- a/tools/tests/test_template_parser.py +++ b/tools/tests/test_template_parser.py @@ -1,7 +1,6 @@ -from typing import Optional - import sys import unittest +from typing import Optional try: from tools.lib.template_parser import ( diff --git a/tools/tests/test_zulint_custom_rules.py b/tools/tests/test_zulint_custom_rules.py index ce11d9ea7d..f858e8de60 100644 --- a/tools/tests/test_zulint_custom_rules.py +++ b/tools/tests/test_zulint_custom_rules.py @@ -1,11 +1,11 @@ -from io import StringIO import os - -from unittest.mock import patch +from io import StringIO from unittest import TestCase +from unittest.mock import patch from zulint.custom_rules import RuleList -from linter_lib.custom_check import python_rules, non_py_rules + +from linter_lib.custom_check import non_py_rules, python_rules ROOT_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..')) CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`." diff --git a/tools/update-prod-static b/tools/update-prod-static index 1acc64781c..c42f45a3a0 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -1,10 +1,8 @@ #!/usr/bin/env python3 # Updates static files for production. - - -import os import argparse +import os import sys # We need settings so we can figure out where the prod-static directory is. @@ -15,11 +13,12 @@ setup_path() os.environ['DJANGO_SETTINGS_MODULE'] = 'zproject.settings' from django.conf import settings -from scripts.lib.node_cache import setup_node_modules -from scripts.lib.zulip_tools import run # check for the venv from lib import sanity_check +from scripts.lib.node_cache import setup_node_modules +from scripts.lib.zulip_tools import run + sanity_check.check_venv(__file__) parser = argparse.ArgumentParser() diff --git a/tools/webpack b/tools/webpack index 1187e0bb42..7ff053b14f 100755 --- a/tools/webpack +++ b/tools/webpack @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import argparse -import os import json +import os import subprocess from typing import NoReturn diff --git a/tools/zulip-export/zulip-export b/tools/zulip-export/zulip-export index 3072588c39..b65dec3eed 100755 --- a/tools/zulip-export/zulip-export +++ b/tools/zulip-export/zulip-export @@ -19,11 +19,11 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. - -import sys -import os import argparse import json +import os +import sys + usage = """Export all messages on a given stream to a JSON dump. zulip-export --user= --api-key= --stream= diff --git a/zerver/apps.py b/zerver/apps.py index d1b95a3827..4d6d355daf 100644 --- a/zerver/apps.py +++ b/zerver/apps.py @@ -6,6 +6,7 @@ from django.conf import settings from django.core.cache import cache from django.db.models.signals import post_migrate + def flush_cache(sender: AppConfig, **kwargs: Any) -> None: logging.info("Clearing memcached cache after migrations") cache.clear() diff --git a/zerver/context_processors.py b/zerver/context_processors.py index a0bda24c60..34bbc538dc 100644 --- a/zerver/context_processors.py +++ b/zerver/context_processors.py @@ -1,27 +1,31 @@ +from typing import Any, Dict, Optional from urllib.parse import urljoin -from typing import Any, Dict, Optional -from django.http import HttpRequest from django.conf import settings +from django.http import HttpRequest -from zerver.models import UserProfile, get_realm, Realm +from version import ( + LATEST_MAJOR_VERSION, + LATEST_RELEASE_ANNOUNCEMENT, + LATEST_RELEASE_VERSION, + ZULIP_VERSION, +) +from zerver.decorator import get_client_name +from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description +from zerver.lib.realm_icon import get_realm_icon_url +from zerver.lib.send_email import FromAddress +from zerver.lib.subdomains import get_subdomain +from zerver.models import Realm, UserProfile, get_realm from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, any_social_backend_enabled, + auth_enabled_helper, get_external_method_dicts, password_auth_enabled, require_email_format_usernames, - auth_enabled_helper, - AUTH_BACKEND_NAME_MAP, - AppleAuthBackend, ) -from zerver.decorator import get_client_name -from zerver.lib.send_email import FromAddress -from zerver.lib.subdomains import get_subdomain -from zerver.lib.realm_icon import get_realm_icon_url -from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description -from version import ZULIP_VERSION, LATEST_RELEASE_VERSION, LATEST_MAJOR_VERSION, \ - LATEST_RELEASE_ANNOUNCEMENT def common_context(user: UserProfile) -> Dict[str, Any]: """Common context used for things like outgoing emails that don't diff --git a/zerver/data_import/gitter.py b/zerver/data_import/gitter.py index 1dd10fa8c2..b168f52854 100644 --- a/zerver/data_import/gitter.py +++ b/zerver/data_import/gitter.py @@ -1,20 +1,31 @@ -import os -import dateutil.parser import logging +import os import subprocess -import ujson +from typing import Any, Dict, List, Set, Tuple +import dateutil.parser +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set, Tuple -from zerver.models import UserProfile, Recipient +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_avatar, + build_defaultstream, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, +) from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, process_avatars, build_realm, build_stream, \ - build_message, create_converted_data_files, make_subscriber_map +from zerver.models import Recipient, UserProfile # stubs GitterDataT = List[Dict[str, Any]] diff --git a/zerver/data_import/hipchat.py b/zerver/data_import/hipchat.py index 92cadb4f3b..386672bf10 100755 --- a/zerver/data_import/hipchat.py +++ b/zerver/data_import/hipchat.py @@ -1,37 +1,29 @@ import base64 -import dateutil import glob -import hypchat import logging import os import re import shutil import subprocess -import ujson - from typing import Any, Callable, Dict, List, Optional, Set +import dateutil +import hypchat +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.utils import ( - process_list_in_batches, -) - -from zerver.models import ( - RealmEmoji, - Recipient, - UserProfile, -) - +from zerver.data_import.hipchat_attachment import AttachmentHandler +from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.import_util import ( + SubscriberHandler, build_message, + build_personal_subscriptions, + build_public_stream_subscriptions, build_realm, build_realm_emoji, build_recipients, build_stream, - build_personal_subscriptions, - build_public_stream_subscriptions, build_stream_subscriptions, build_user_profile, build_zerver_realm, @@ -39,12 +31,10 @@ from zerver.data_import.import_util import ( make_subscriber_map, make_user_messages, write_avatar_png, - SubscriberHandler, ) - -from zerver.data_import.hipchat_attachment import AttachmentHandler -from zerver.data_import.hipchat_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.utils import process_list_in_batches +from zerver.models import RealmEmoji, Recipient, UserProfile # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/hipchat_attachment.py b/zerver/data_import/hipchat_attachment.py index de676751dd..596d3e588f 100644 --- a/zerver/data_import/hipchat_attachment.py +++ b/zerver/data_import/hipchat_attachment.py @@ -1,14 +1,11 @@ import logging -import shutil import os - -from zerver.data_import.import_util import ( - build_attachment, - create_converted_data_files, -) - +import shutil from typing import Any, Dict, List, Optional +from zerver.data_import.import_util import build_attachment, create_converted_data_files + + class AttachmentHandler: def __init__(self) -> None: self.info_dict: Dict[str, Dict[str, Any]] = dict() diff --git a/zerver/data_import/hipchat_user.py b/zerver/data_import/hipchat_user.py index 10b3950dee..ef03ece81b 100644 --- a/zerver/data_import/hipchat_user.py +++ b/zerver/data_import/hipchat_user.py @@ -2,12 +2,9 @@ from typing import Any, Dict, List from django.utils.timezone import now as timezone_now -from zerver.data_import.import_util import ( - build_user_profile, -) -from zerver.models import ( - UserProfile, -) +from zerver.data_import.import_util import build_user_profile +from zerver.models import UserProfile + class UserHandler: ''' diff --git a/zerver/data_import/import_util.py b/zerver/data_import/import_util.py index f7580e454e..31f235df09 100644 --- a/zerver/data_import/import_util.py +++ b/zerver/data_import/import_util.py @@ -1,20 +1,29 @@ -import random -import requests -import shutil import logging import os +import random +import shutil import traceback -import ujson +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar -from typing import List, Dict, Any, Optional, Set, Callable, Iterable, Tuple, TypeVar +import requests +import ujson from django.forms.models import model_to_dict -from zerver.models import Realm, RealmEmoji, Subscription, Recipient, \ - Attachment, Stream, Message, UserProfile, Huddle from zerver.data_import.sequencer import NEXT_ID from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS as stream_colors from zerver.lib.avatar_hash import user_avatar_path_from_ids from zerver.lib.parallel import run_parallel +from zerver.models import ( + Attachment, + Huddle, + Message, + Realm, + RealmEmoji, + Recipient, + Stream, + Subscription, + UserProfile, +) # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py index 1bb4dfa3be..ba1823a4fc 100644 --- a/zerver/data_import/mattermost.py +++ b/zerver/data_import/mattermost.py @@ -2,33 +2,42 @@ spec: https://docs.mattermost.com/administration/bulk-export.html """ -import os import logging -import subprocess -import ujson +import os import re import shutil - +import subprocess from typing import Any, Callable, Dict, List, Set +import ujson from django.conf import settings -from django.utils.timezone import now as timezone_now from django.forms.models import model_to_dict +from django.utils.timezone import now as timezone_now -from zerver.models import Recipient, RealmEmoji, Reaction, UserProfile -from zerver.lib.utils import ( - process_list_in_batches, +from zerver.data_import.import_util import ( + SubscriberHandler, + ZerverFieldsT, + build_huddle, + build_huddle_subscriptions, + build_message, + build_personal_subscriptions, + build_realm, + build_realm_emoji, + build_recipients, + build_stream, + build_stream_subscriptions, + build_user_profile, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + make_user_messages, ) -from zerver.lib.emoji import name_to_codepoint -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_stream, build_realm, build_message, create_converted_data_files, \ - make_subscriber_map, build_recipients, build_user_profile, \ - build_stream_subscriptions, build_huddle_subscriptions, \ - build_personal_subscriptions, SubscriberHandler, \ - build_realm_emoji, make_user_messages, build_huddle - from zerver.data_import.mattermost_user import UserHandler from zerver.data_import.sequencer import NEXT_ID, IdMapper +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.utils import process_list_in_batches +from zerver.models import Reaction, RealmEmoji, Recipient, UserProfile + def make_realm(realm_id: int, team: Dict[str, Any]) -> ZerverFieldsT: # set correct realm details diff --git a/zerver/data_import/mattermost_user.py b/zerver/data_import/mattermost_user.py index c8585584a1..e6d629314e 100644 --- a/zerver/data_import/mattermost_user.py +++ b/zerver/data_import/mattermost_user.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List + class UserHandler: ''' Our UserHandler class is a glorified wrapper diff --git a/zerver/data_import/slack.py b/zerver/data_import/slack.py index 71b487dde7..798345cd8b 100755 --- a/zerver/data_import/slack.py +++ b/zerver/data_import/slack.py @@ -1,33 +1,55 @@ +import logging import os -import ujson +import random import shutil import subprocess -import logging -import random -import requests - from collections import defaultdict - -from django.conf import settings -from django.utils.timezone import now as timezone_now -from django.forms.models import model_to_dict -from typing import Any, Dict, List, Optional, Tuple, Set, Iterator -from zerver.models import Reaction, RealmEmoji, UserProfile, Recipient, \ - CustomProfileField, CustomProfileFieldValue, Realm -from zerver.data_import.slack_message_conversion import convert_to_zulip_markdown, \ - get_user_full_name -from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ - build_avatar, build_subscription, build_recipient, build_usermessages, \ - build_defaultstream, build_attachment, process_avatars, process_uploads, \ - process_emojis, build_realm, build_stream, build_huddle, build_message, \ - create_converted_data_files, make_subscriber_map -from zerver.data_import.sequencer import NEXT_ID -from zerver.lib.upload import random_name, sanitize_name -from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE -from zerver.lib.emoji import name_to_codepoint -from zerver.lib.upload import resize_logo +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple from urllib.parse import urlencode +import requests +import ujson +from django.conf import settings +from django.forms.models import model_to_dict +from django.utils.timezone import now as timezone_now + +from zerver.data_import.import_util import ( + ZerverFieldsT, + build_attachment, + build_avatar, + build_defaultstream, + build_huddle, + build_message, + build_realm, + build_recipient, + build_stream, + build_subscription, + build_usermessages, + build_zerver_realm, + create_converted_data_files, + make_subscriber_map, + process_avatars, + process_emojis, + process_uploads, +) +from zerver.data_import.sequencer import NEXT_ID +from zerver.data_import.slack_message_conversion import ( + convert_to_zulip_markdown, + get_user_full_name, +) +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE +from zerver.lib.upload import random_name, resize_logo, sanitize_name +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Reaction, + Realm, + RealmEmoji, + Recipient, + UserProfile, +) + SlackToZulipUserIDT = Dict[str, int] AddedChannelsT = Dict[str, Tuple[str, int]] AddedMPIMsT = Dict[str, Tuple[str, int]] diff --git a/zerver/data_import/slack_message_conversion.py b/zerver/data_import/slack_message_conversion.py index 0905541568..bfd8a54eb2 100644 --- a/zerver/data_import/slack_message_conversion.py +++ b/zerver/data_import/slack_message_conversion.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, Tuple, List, Optional +from typing import Any, Dict, List, Optional, Tuple # stubs ZerverFieldsT = Dict[str, Any] diff --git a/zerver/decorator.py b/zerver/decorator.py index f87eacdccc..ae4a62d231 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -1,51 +1,60 @@ -import django_otp -from two_factor.utils import default_device -from django_otp import user_has_device - -from django.contrib.auth.decorators import user_passes_test as django_user_passes_test -from django.contrib.auth.models import AnonymousUser -from django.utils.translation import ugettext as _ -from django.http import HttpResponseRedirect, HttpResponse -from django.contrib.auth import REDIRECT_FIELD_NAME, login as django_login -from django.views.decorators.csrf import csrf_exempt -from django.http import QueryDict, HttpResponseNotAllowed, HttpRequest -from django.http.multipartparser import MultiPartParser -from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key -from zerver.lib.response import json_error, json_unauthorized, json_success -from django.shortcuts import resolve_url -from django.utils.decorators import available_attrs -from django.utils.timezone import now as timezone_now -from django.conf import settings -from django.template.response import SimpleTemplateResponse - -from zerver.lib.exceptions import UnexpectedWebhookEventType -from zerver.lib.queue import queue_json_publish -from zerver.lib.subdomains import get_subdomain, user_matches_subdomain -from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime -from zerver.lib.utils import statsd, has_api_key_format -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError, InvalidAPIKeyError, InvalidAPIKeyFormatError, \ - OrganizationAdministratorRequired, OrganizationOwnerRequired -from zerver.lib.types import ViewFuncT - -from zerver.lib.rate_limiter import RateLimitedUser -from zerver.lib.request import REQ, has_request_variables - -from functools import wraps import base64 import datetime -import ujson import logging -from io import BytesIO import urllib +from functools import wraps +from io import BytesIO +from typing import Any, Callable, Dict, Optional, Tuple, TypeVar, Union -from typing import Union, Any, Callable, Dict, Optional, TypeVar, Tuple +import django_otp +import ujson +from django.conf import settings +from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.auth import login as django_login +from django.contrib.auth.decorators import user_passes_test as django_user_passes_test +from django.contrib.auth.models import AnonymousUser +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseNotAllowed, + HttpResponseRedirect, + QueryDict, +) +from django.http.multipartparser import MultiPartParser +from django.shortcuts import resolve_url +from django.template.response import SimpleTemplateResponse +from django.utils.decorators import available_attrs +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import csrf_exempt +from django_otp import user_has_device +from two_factor.utils import default_device + +from zerver.lib.exceptions import ( + ErrorCode, + InvalidAPIKeyError, + InvalidAPIKeyFormatError, + InvalidJSONError, + JsonableError, + OrganizationAdministratorRequired, + OrganizationOwnerRequired, + UnexpectedWebhookEventType, +) from zerver.lib.logging_util import log_to_file +from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedUser +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success, json_unauthorized +from zerver.lib.subdomains import get_subdomain, user_matches_subdomain +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.lib.types import ViewFuncT +from zerver.lib.utils import has_api_key_format, statsd +from zerver.models import Realm, UserProfile, get_client, get_user_profile_by_api_key # This is a hack to ensure that RemoteZulipServer always exists even # if Zilencer isn't enabled. if settings.ZILENCER_ENABLED: - from zilencer.models import get_remote_server_by_uuid, RemoteZulipServer + from zilencer.models import RemoteZulipServer, get_remote_server_by_uuid else: # nocoverage # Hack here basically to make impossible code paths compile from unittest.mock import Mock get_remote_server_by_uuid = Mock() @@ -130,6 +139,7 @@ def require_billing_access(func: ViewFuncT) -> ViewFuncT: from zerver.lib.user_agent import parse_user_agent + def get_client_name(request: HttpRequest) -> str: # If the API request specified a client in the request content, # that has priority. Otherwise, extract the client from the diff --git a/zerver/filters.py b/zerver/filters.py index f74eae3aae..8d3e2fbe47 100644 --- a/zerver/filters.py +++ b/zerver/filters.py @@ -4,6 +4,7 @@ from typing import Any, Dict from django.http import HttpRequest from django.views.debug import SafeExceptionReporterFilter + class ZulipExceptionReporterFilter(SafeExceptionReporterFilter): def get_post_parameters(self, request: HttpRequest) -> Dict[str, Any]: filtered_post = SafeExceptionReporterFilter.get_post_parameters(self, request).copy() diff --git a/zerver/forms.py b/zerver/forms.py index 6b70e7f67c..2f1cc9ca1d 100644 --- a/zerver/forms.py +++ b/zerver/forms.py @@ -1,42 +1,44 @@ +import logging +import re +from typing import Any, Dict, List, Optional, Tuple + +import DNS from django import forms from django.conf import settings from django.contrib.auth import authenticate, password_validation -from django.contrib.auth.forms import SetPasswordForm, AuthenticationForm, \ - PasswordResetForm +from django.contrib.auth.forms import AuthenticationForm, PasswordResetForm, SetPasswordForm +from django.contrib.auth.tokens import PasswordResetTokenGenerator, default_token_generator from django.core.exceptions import ValidationError -from django.urls import reverse from django.core.validators import validate_email -from django.utils.translation import ugettext as _ -from django.contrib.auth.tokens import default_token_generator -from django.utils.http import urlsafe_base64_encode -from django.utils.encoding import force_bytes -from django.contrib.auth.tokens import PasswordResetTokenGenerator from django.http import HttpRequest +from django.urls import reverse +from django.utils.encoding import force_bytes +from django.utils.http import urlsafe_base64_encode +from django.utils.translation import ugettext as _ from jinja2 import Markup as mark_safe - -from zerver.lib.actions import do_change_password, email_not_system_bot -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.name_restrictions import is_reserved_subdomain, is_disposable_domain -from zerver.lib.rate_limiter import RateLimited, RateLimitedObject -from zerver.lib.request import JsonableError -from zerver.lib.send_email import send_email, FromAddress -from zerver.lib.subdomains import get_subdomain, is_root_domain_available -from zerver.lib.users import check_full_name -from zerver.models import Realm, get_user_by_delivery_email, UserProfile, get_realm, \ - email_to_domain, \ - DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError -from zproject.backends import email_auth_enabled, email_belongs_to_ldap, check_password_strength - -import logging -import re -import DNS - -from typing import Any, List, Optional, Dict, Tuple from two_factor.forms import AuthenticationTokenForm as TwoFactorAuthenticationTokenForm from two_factor.utils import totp_digits +from zerver.lib.actions import do_change_password, email_not_system_bot +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm +from zerver.lib.name_restrictions import is_disposable_domain, is_reserved_subdomain +from zerver.lib.rate_limiter import RateLimited, RateLimitedObject +from zerver.lib.request import JsonableError +from zerver.lib.send_email import FromAddress, send_email +from zerver.lib.subdomains import get_subdomain, is_root_domain_available +from zerver.lib.users import check_full_name +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + Realm, + UserProfile, + email_to_domain, + get_realm, + get_user_by_delivery_email, +) +from zproject.backends import check_password_strength, email_auth_enabled, email_belongs_to_ldap + MIT_VALIDATION_ERROR = 'That user does not exist at MIT or is a ' + \ 'mailing list. ' + \ 'If you want to sign up an alias for Zulip, ' + \ diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 42902cf44b..6bce33df3a 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -1,63 +1,116 @@ from typing import ( - AbstractSet, Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, - Optional, Sequence, Set, Tuple, Union, cast, + AbstractSet, + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Tuple, + Union, + cast, ) -from typing_extensions import TypedDict import django.db.utils -from django.db.models import Count, Exists, OuterRef -from django.contrib.contenttypes.models import ContentType -from django.utils.html import escape -from django.utils.translation import ugettext as _ from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError from django.core.files import File +from django.db import IntegrityError, connection, transaction +from django.db.models import Count, Exists, F, Max, OuterRef, Q, Sum +from django.db.models.query import QuerySet +from django.utils.html import escape +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ from psycopg2.extras import execute_values from psycopg2.sql import SQL -from analytics.lib.counts import COUNT_STATS, do_increment_logging_stat, \ - RealmCount +from typing_extensions import TypedDict -from zerver.lib.bugdown import ( - version as bugdown_version, +from analytics.lib.counts import COUNT_STATS, RealmCount, do_increment_logging_stat +from analytics.models import StreamCount +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + confirmation_url, + create_confirmation_link, + generate_key, ) +from zerver.decorator import statsd_increment +from zerver.lib import bugdown from zerver.lib.addressee import Addressee -from zerver.lib.bot_config import ( - ConfigError, - get_bot_config, - get_bot_configs, - set_bot_config, +from zerver.lib.alert_words import ( + add_user_alert_words, + get_alert_word_automaton, + remove_user_alert_words, ) +from zerver.lib.avatar import avatar_url, avatar_url_from_dict +from zerver.lib.bot_config import ConfigError, get_bot_config, get_bot_configs, set_bot_config +from zerver.lib.bugdown import version as bugdown_version +from zerver.lib.bulk_create import bulk_create_users from zerver.lib.cache import ( bot_dict_fields, - display_recipient_cache_key, + cache_delete, + cache_delete_many, + cache_set, + cache_set_many, + cache_with_key, delete_user_profile_caches, + display_recipient_cache_key, flush_user_profile, to_dict_cache_key_id, user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, ) from zerver.lib.context_managers import lockfile +from zerver.lib.create_user import create_user, get_display_email_address from zerver.lib.email_mirror_helpers import encode_email_address, encode_email_address_helper +from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.email_validation import ( + email_reserved_for_system_bots_error, + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.emoji import emoji_name_to_emoji_code, get_emoji_file_name -from zerver.lib.exceptions import StreamDoesNotExistError, \ - StreamWithIDDoesNotExistError +from zerver.lib.exceptions import ( + BugdownRenderingException, + ErrorCode, + JsonableError, + StreamDoesNotExistError, + StreamWithIDDoesNotExistError, +) from zerver.lib.export import get_realm_exports_serialized from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots +from zerver.lib.i18n import get_language_name from zerver.lib.message import ( - access_message, MessageDict, + access_message, render_markdown, - update_first_visible_message_id, truncate_body, truncate_topic, + update_first_visible_message_id, ) from zerver.lib.pysa import mark_sanitized +from zerver.lib.queue import queue_json_publish from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_data from zerver.lib.retention import move_messages_to_archive -from zerver.lib.send_email import send_email, FromAddress, send_email_to_admins, \ - clear_scheduled_emails, clear_scheduled_invitation_emails +from zerver.lib.send_email import ( + FromAddress, + clear_scheduled_emails, + clear_scheduled_invitation_emails, + send_email, + send_email_to_admins, +) from zerver.lib.server_initialization import create_internal_realm, server_initialized +from zerver.lib.sessions import delete_user_sessions from zerver.lib.storage import static_path +from zerver.lib.stream_recipient import StreamRecipientMap from zerver.lib.stream_subscription import ( get_active_subscriptions_for_stream_id, get_active_subscriptions_for_stream_ids, @@ -68,21 +121,37 @@ from zerver.lib.stream_subscription import ( num_subscribers_for_stream_id, ) from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.streams import ( + access_stream_for_send_message, + check_stream_name, + create_stream_if_needed, + get_default_value_for_history_public_to_subscribers, + render_stream_description, + send_stream_creation_event, + subscribed_to_stream, +) +from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.topic import ( + LEGACY_PREV_TOPIC, + ORIG_TOPIC, + TOPIC_LINKS, + TOPIC_NAME, filter_by_exact_message_topic, filter_by_topic_name_via_message, save_message_for_edit_use_case, update_messages_for_topic_edit, - ORIG_TOPIC, - LEGACY_PREV_TOPIC, - TOPIC_LINKS, - TOPIC_NAME, ) -from zerver.lib.topic_mutes import ( - get_topic_mutes, - add_topic_mute, - remove_topic_mute, +from zerver.lib.topic_mutes import add_topic_mute, get_topic_mutes, remove_topic_mute +from zerver.lib.types import ProfileFieldData +from zerver.lib.upload import ( + claim_attachment, + delete_avatar_image, + delete_export_tarball, + delete_message_image, + upload_emoji_image, ) +from zerver.lib.user_groups import access_user_group_by_id, create_user_group +from zerver.lib.user_status import update_user_status from zerver.lib.users import ( check_bot_name_available, check_full_name, @@ -90,93 +159,84 @@ from zerver.lib.users import ( get_api_key, user_profile_to_user_row, ) -from zerver.lib.user_status import ( - update_user_status, -) -from zerver.lib.user_groups import create_user_group, access_user_group_by_id - -from zerver.models import Realm, RealmEmoji, Stream, UserProfile, UserActivity, \ - RealmDomain, Service, SubMessage, \ - Subscription, Recipient, Message, Attachment, UserMessage, RealmAuditLog, \ - UserHotspot, MultiuseInvite, ScheduledMessage, UserStatus, \ - Client, DefaultStream, DefaultStreamGroup, UserPresence, \ - ScheduledEmail, \ - MAX_MESSAGE_LENGTH, get_client, get_stream, \ - get_user_profile_by_id, PreregistrationUser, \ - email_to_username, \ - get_user_by_delivery_email, get_stream_cache_key, active_non_guest_user_ids, \ - UserActivityInterval, active_user_ids, get_active_streams, \ - realm_filters_for_realm, RealmFilter, stream_name_in_use, \ - get_old_unclaimed_attachments, is_cross_realm_bot_email, \ - Reaction, EmailChangeStatus, CustomProfileField, \ - custom_profile_fields_for_realm, get_huddle_user_ids, \ - CustomProfileFieldValue, validate_attachment_request, get_system_bot, \ - query_for_ids, get_huddle_recipient, \ - UserGroup, UserGroupMembership, get_default_stream_groups, \ - get_bot_services, get_bot_dicts_in_realm, \ - get_user_by_id_in_realm_including_cross_realm, \ - get_stream_by_id_in_realm - -from zerver.lib.alert_words import get_alert_word_automaton -from zerver.lib.avatar import avatar_url, avatar_url_from_dict -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors, \ - email_reserved_for_system_bots_error -from zerver.lib.stream_recipient import StreamRecipientMap +from zerver.lib.utils import generate_api_key, log_statsd_event, statsd from zerver.lib.validator import check_widget_content from zerver.lib.widget import do_widget_post_save_actions - -from django.db import transaction, IntegrityError, connection -from django.db.models import F, Q, Max, Sum -from django.db.models.query import QuerySet -from django.core.exceptions import ValidationError -from django.utils.timezone import now as timezone_now - -from confirmation.models import Confirmation, create_confirmation_link, generate_key, \ - confirmation_url -from confirmation import settings as confirmation_settings - -from zerver.lib.bulk_create import bulk_create_users -from zerver.lib.timestamp import timestamp_to_datetime, datetime_to_timestamp -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_api_key -from zerver.lib.create_user import create_user, get_display_email_address -from zerver.lib import bugdown -from zerver.lib.cache import cache_with_key, cache_set, \ - user_profile_by_email_cache_key, \ - cache_set_many, cache_delete, cache_delete_many -from zerver.decorator import statsd_increment -from zerver.lib.utils import log_statsd_event, statsd -from zerver.lib.i18n import get_language_name -from zerver.lib.alert_words import add_user_alert_words, \ - remove_user_alert_words -from zerver.lib.email_notifications import enqueue_welcome_emails -from zerver.lib.exceptions import JsonableError, ErrorCode, BugdownRenderingException -from zerver.lib.sessions import delete_user_sessions -from zerver.lib.upload import claim_attachment, delete_message_image, \ - upload_emoji_image, delete_avatar_image, \ - delete_export_tarball +from zerver.models import ( + MAX_MESSAGE_LENGTH, + Attachment, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + DefaultStreamGroup, + EmailChangeStatus, + Message, + MultiuseInvite, + PreregistrationUser, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + ScheduledEmail, + ScheduledMessage, + Service, + Stream, + SubMessage, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + UserStatus, + active_non_guest_user_ids, + active_user_ids, + custom_profile_fields_for_realm, + email_to_username, + get_active_streams, + get_bot_dicts_in_realm, + get_bot_services, + get_client, + get_default_stream_groups, + get_huddle_recipient, + get_huddle_user_ids, + get_old_unclaimed_attachments, + get_stream, + get_stream_by_id_in_realm, + get_stream_cache_key, + get_system_bot, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_profile_by_id, + is_cross_realm_bot_email, + query_for_ids, + realm_filters_for_realm, + stream_name_in_use, + validate_attachment_request, +) from zerver.tornado.event_queue import send_event -from zerver.lib.types import ProfileFieldData -from zerver.lib.streams import access_stream_for_send_message, subscribed_to_stream, check_stream_name, \ - create_stream_if_needed, get_default_value_for_history_public_to_subscribers, \ - render_stream_description, send_stream_creation_event - -from analytics.models import StreamCount if settings.BILLING_ENABLED: from corporate.lib.stripe import update_license_ledger_if_needed, downgrade_now -import ujson -import time import datetime +import itertools +import logging import os import platform -import logging -import itertools +import time from collections import defaultdict from operator import itemgetter +import ujson + # This will be used to type annotate parameters in a function if the function # works on both str and unicode in python 2 but in python 3 it only works on str. SizedTextIterable = Union[Sequence[str], AbstractSet[str]] diff --git a/zerver/lib/addressee.py b/zerver/lib/addressee.py index cb3dab8789..ec1c6fd7a9 100644 --- a/zerver/lib/addressee.py +++ b/zerver/lib/addressee.py @@ -1,15 +1,17 @@ from typing import Iterable, List, Optional, Sequence, Union, cast from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError from zerver.models import ( Realm, - UserProfile, - get_user_including_cross_realm, - get_user_by_id_in_realm_including_cross_realm, Stream, + UserProfile, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, ) + def get_user_profiles(emails: Iterable[str], realm: Realm) -> List[UserProfile]: user_profiles: List[UserProfile] = [] for email in emails: diff --git a/zerver/lib/alert_words.py b/zerver/lib/alert_words.py index 51c4c49794..e9658d9b97 100644 --- a/zerver/lib/alert_words.py +++ b/zerver/lib/alert_words.py @@ -1,10 +1,15 @@ +from typing import Dict, Iterable, List + +import ahocorasick from django.db import transaction -from zerver.models import UserProfile, Realm, AlertWord, flush_realm_alert_words -from zerver.lib.cache import cache_with_key, realm_alert_words_cache_key, \ - realm_alert_words_automaton_cache_key -import ahocorasick -from typing import Dict, Iterable, List +from zerver.lib.cache import ( + cache_with_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, +) +from zerver.models import AlertWord, Realm, UserProfile, flush_realm_alert_words + @cache_with_key(realm_alert_words_cache_key, timeout=3600*24) def alert_words_in_realm(realm: Realm) -> Dict[int, List[str]]: diff --git a/zerver/lib/attachments.py b/zerver/lib/attachments.py index ededc38cbb..477770f658 100644 --- a/zerver/lib/attachments.py +++ b/zerver/lib/attachments.py @@ -1,10 +1,12 @@ -from django.utils.translation import ugettext as _ from typing import Any, Dict, List +from django.utils.translation import ugettext as _ + from zerver.lib.request import JsonableError from zerver.lib.upload import delete_message_image from zerver.models import Attachment, UserProfile + def user_attachments(user_profile: UserProfile) -> List[Dict[str, Any]]: attachments = Attachment.objects.filter(owner=user_profile).prefetch_related('messages') return [a.to_dict() for a in attachments] diff --git a/zerver/lib/avatar.py b/zerver/lib/avatar.py index 469b8e27f8..c6bddfeb45 100644 --- a/zerver/lib/avatar.py +++ b/zerver/lib/avatar.py @@ -1,11 +1,16 @@ -from django.conf import settings - +import urllib from typing import Any, Dict, Optional -from zerver.lib.avatar_hash import gravatar_hash, user_avatar_path_from_ids, user_avatar_content_hash -from zerver.lib.upload import upload_backend, MEDIUM_AVATAR_SIZE +from django.conf import settings + +from zerver.lib.avatar_hash import ( + gravatar_hash, + user_avatar_content_hash, + user_avatar_path_from_ids, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, upload_backend from zerver.models import UserProfile -import urllib + def avatar_url(user_profile: UserProfile, medium: bool=False, client_gravatar: bool=False) -> Optional[str]: diff --git a/zerver/lib/avatar_hash.py b/zerver/lib/avatar_hash.py index bbd6c10c17..492a575cc0 100644 --- a/zerver/lib/avatar_hash.py +++ b/zerver/lib/avatar_hash.py @@ -1,10 +1,10 @@ +import hashlib + from django.conf import settings from zerver.lib.utils import make_safe_digest - from zerver.models import UserProfile -import hashlib def gravatar_hash(email: str) -> str: """Compute the Gravatar hash for an email address.""" diff --git a/zerver/lib/bot_config.py b/zerver/lib/bot_config.py index caf6cd2059..40e74559cc 100644 --- a/zerver/lib/bot_config.py +++ b/zerver/lib/bot_config.py @@ -1,17 +1,16 @@ -from django.conf import settings -from django.db.models import Sum -from django.db.models.query import F -from django.db.models.functions import Length -from zerver.models import BotConfigData, UserProfile - -from typing import List, Dict, Optional - -from collections import defaultdict - -import os - import configparser import importlib +import os +from collections import defaultdict +from typing import Dict, List, Optional + +from django.conf import settings +from django.db.models import Sum +from django.db.models.functions import Length +from django.db.models.query import F + +from zerver.models import BotConfigData, UserProfile + class ConfigError(Exception): pass diff --git a/zerver/lib/bot_lib.py b/zerver/lib/bot_lib.py index 492dccd9cf..b25b96f521 100644 --- a/zerver/lib/bot_lib.py +++ b/zerver/lib/bot_lib.py @@ -1,23 +1,31 @@ +import importlib import json import os -import importlib -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message_by_name, internal_send_huddle_message -from zerver.models import UserProfile, get_active_user -from zerver.lib.bot_storage import get_bot_storage, set_bot_storage, \ - is_key_in_bot_storage, remove_bot_storage -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.topic import get_topic_from_message_info +from typing import Any, Dict from django.utils.translation import ugettext as _ -from typing import Any, Dict +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message_by_name, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_storage import ( + get_bot_storage, + is_key_in_bot_storage, + remove_bot_storage, + set_bot_storage, +) +from zerver.lib.integrations import EMBEDDED_BOTS +from zerver.lib.topic import get_topic_from_message_info +from zerver.models import UserProfile, get_active_user our_dir = os.path.dirname(os.path.abspath(__file__)) from zulip_bots.lib import RateLimit + def get_bot_handler(service_name: str) -> Any: # Check that this service is present in EMBEDDED_BOTS, add exception handling. diff --git a/zerver/lib/bot_storage.py b/zerver/lib/bot_storage.py index ba3a9b1af3..1b43644f0b 100644 --- a/zerver/lib/bot_storage.py +++ b/zerver/lib/bot_storage.py @@ -1,10 +1,12 @@ +from typing import List, Optional, Tuple + from django.conf import settings from django.db.models import Sum -from django.db.models.query import F from django.db.models.functions import Length +from django.db.models.query import F + from zerver.models import BotStorageData, UserProfile -from typing import Optional, List, Tuple class StateError(Exception): pass diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index bfab78823a..a0bcdc85cc 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -1,65 +1,76 @@ # Zulip's main markdown implementation. See docs/subsystems/markdown.md for # detailed documentation on our markdown syntax. -from typing import (Any, Callable, Dict, Generic, Iterable, List, - Optional, Set, Tuple, TypeVar, Union) -from typing.re import Match, Pattern -from typing_extensions import TypedDict - -import markdown +import functools +import html import logging +import os +import re +import time import traceback import urllib import urllib.parse -import re -import os -import html -import time -import functools +from collections import defaultdict, deque from dataclasses import dataclass +from datetime import datetime from io import StringIO +from typing import ( + Any, + Callable, + Dict, + Generic, + Iterable, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from typing.re import Match, Pattern +from xml.etree import ElementTree as etree +from xml.etree.ElementTree import Element, SubElement + +import ahocorasick import dateutil.parser import dateutil.tz -from datetime import datetime -import xml.etree.ElementTree as etree -from xml.etree.ElementTree import Element, SubElement -import ahocorasick -from hyperlink import parse - -from collections import deque, defaultdict - +import markdown import requests - from django.conf import settings from django.db.models import Q +from hyperlink import parse +from markdown.extensions import codehilite, nl2br, sane_lists, tables +from typing_extensions import TypedDict -from markdown.extensions import codehilite, nl2br, tables, sane_lists +from zerver.lib import mention as mention from zerver.lib.bugdown import fenced_code from zerver.lib.bugdown.fenced_code import FENCE_RE +from zerver.lib.cache import NotFoundInCache, cache_with_key from zerver.lib.camo import get_camo_url -from zerver.lib.emoji import translate_emoticons, emoticon_regex, \ - name_to_codepoint, codepoint_to_name -from zerver.lib.mention import possible_mentions, \ - possible_user_group_mentions, extract_user_group -from zerver.lib.url_encoding import encode_stream, hash_util_encode +from zerver.lib.emoji import ( + codepoint_to_name, + emoticon_regex, + name_to_codepoint, + translate_emoticons, +) +from zerver.lib.exceptions import BugdownRenderingException +from zerver.lib.mention import extract_user_group, possible_mentions, possible_user_group_mentions +from zerver.lib.tex import render_tex from zerver.lib.thumbnail import user_uploads_or_external -from zerver.lib.timeout import timeout, TimeoutExpired -from zerver.lib.cache import cache_with_key, NotFoundInCache +from zerver.lib.timeout import TimeoutExpired, timeout +from zerver.lib.timezone import get_common_timezones +from zerver.lib.url_encoding import encode_stream, hash_util_encode from zerver.lib.url_preview import preview as link_preview from zerver.models import ( - all_realm_filters, - get_active_streams, MAX_MESSAGE_LENGTH, Message, Realm, - realm_filters_for_realm, - UserProfile, UserGroup, UserGroupMembership, + UserProfile, + all_realm_filters, + get_active_streams, + realm_filters_for_realm, ) -import zerver.lib.mention as mention -from zerver.lib.tex import render_tex -from zerver.lib.exceptions import BugdownRenderingException -from zerver.lib.timezone import get_common_timezones ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/bugdown/api_arguments_table_generator.py b/zerver/lib/bugdown/api_arguments_table_generator.py index a9209f7b05..5aab676008 100644 --- a/zerver/lib/bugdown/api_arguments_table_generator.py +++ b/zerver/lib/bugdown/api_arguments_table_generator.py @@ -1,13 +1,14 @@ -import re -import os import json +import os +import re +from typing import Any, Dict, List, Optional +import markdown from django.utils.html import escape as escape_html from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_parameters -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/api_return_values_table_generator.py b/zerver/lib/bugdown/api_return_values_table_generator.py index 865b74006f..9c56ac3790 100644 --- a/zerver/lib/bugdown/api_return_values_table_generator.py +++ b/zerver/lib/bugdown/api_return_values_table_generator.py @@ -1,10 +1,11 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor + from zerver.openapi.openapi import get_openapi_return_values -from typing import Any, Dict, Optional, List -import markdown REGEXP = re.compile(r'\{generate_return_values_table\|\s*(.+?)\s*\|\s*(.+)\s*\}') diff --git a/zerver/lib/bugdown/fenced_code.py b/zerver/lib/bugdown/fenced_code.py index 6eb45bb1db..fe35b6a5f6 100644 --- a/zerver/lib/bugdown/fenced_code.py +++ b/zerver/lib/bugdown/fenced_code.py @@ -75,14 +75,15 @@ Dependencies: * [Pygments (optional)](http://pygments.org) """ - import re +from typing import Any, Dict, Iterable, List, MutableSequence, Optional + import markdown from django.utils.html import escape from markdown.extensions.codehilite import CodeHilite, CodeHiliteExtension + from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.tex import render_tex -from typing import Any, Dict, Iterable, List, MutableSequence, Optional # Global vars FENCE_RE = re.compile(""" diff --git a/zerver/lib/bugdown/help_emoticon_translations_table.py b/zerver/lib/bugdown/help_emoticon_translations_table.py index 26e84e5157..079933c6b7 100644 --- a/zerver/lib/bugdown/help_emoticon_translations_table.py +++ b/zerver/lib/bugdown/help_emoticon_translations_table.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor from zerver.lib.emoji import EMOTICON_CONVERSIONS, name_to_codepoint diff --git a/zerver/lib/bugdown/help_relative_links.py b/zerver/lib/bugdown/help_relative_links.py index 910d908471..1f086b7cab 100644 --- a/zerver/lib/bugdown/help_relative_links.py +++ b/zerver/lib/bugdown/help_relative_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/help_settings_links.py b/zerver/lib/bugdown/help_settings_links.py index 4c7b9e34e3..f60cccb116 100644 --- a/zerver/lib/bugdown/help_settings_links.py +++ b/zerver/lib/bugdown/help_settings_links.py @@ -1,7 +1,8 @@ import re -import markdown from typing import Any, Dict, List, Optional from typing.re import Match + +import markdown from markdown.preprocessors import Preprocessor # There is a lot of duplicated code between this file and diff --git a/zerver/lib/bugdown/include.py b/zerver/lib/bugdown/include.py index 0d3ba9938a..00536f56c9 100644 --- a/zerver/lib/bugdown/include.py +++ b/zerver/lib/bugdown/include.py @@ -1,9 +1,9 @@ -import re import os +import re from typing import Any, Dict, List import markdown -from markdown_include.include import MarkdownInclude, IncludePreprocessor +from markdown_include.include import IncludePreprocessor, MarkdownInclude from zerver.lib.exceptions import InvalidMarkdownIncludeStatement diff --git a/zerver/lib/bugdown/nested_code_blocks.py b/zerver/lib/bugdown/nested_code_blocks.py index 92a457e6c2..02f16dd05d 100644 --- a/zerver/lib/bugdown/nested_code_blocks.py +++ b/zerver/lib/bugdown/nested_code_blocks.py @@ -1,9 +1,11 @@ -from markdown.extensions import Extension -from typing import Any, Dict, Optional, List, Tuple -import markdown +from typing import Any, Dict, List, Optional, Tuple from xml.etree.ElementTree import Element, SubElement -from zerver.lib.bugdown import walk_tree_with_family, ResultWithFamily +import markdown +from markdown.extensions import Extension + +from zerver.lib.bugdown import ResultWithFamily, walk_tree_with_family + class NestedCodeBlocksRenderer(Extension): def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: diff --git a/zerver/lib/bugdown/tabbed_sections.py b/zerver/lib/bugdown/tabbed_sections.py index bd38d4e9ed..3cea656852 100644 --- a/zerver/lib/bugdown/tabbed_sections.py +++ b/zerver/lib/bugdown/tabbed_sections.py @@ -1,9 +1,9 @@ import re +from typing import Any, Dict, List, Optional +import markdown from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List -import markdown START_TABBED_SECTION_REGEX = re.compile(r'^\{start_tabs\}$') END_TABBED_SECTION_REGEX = re.compile(r'^\{end_tabs\}$') diff --git a/zerver/lib/bugdown/testing_mocks.py b/zerver/lib/bugdown/testing_mocks.py index f47e9dd57a..04677c36d6 100644 --- a/zerver/lib/bugdown/testing_mocks.py +++ b/zerver/lib/bugdown/testing_mocks.py @@ -1,6 +1,6 @@ from typing import Any, Dict, Optional -import ujson +import ujson NORMAL_TWEET = """{ "created_at": "Sat Sep 10 22:23:38 +0000 2011", diff --git a/zerver/lib/bulk_create.py b/zerver/lib/bulk_create.py index 5d05681bcc..c988197827 100644 --- a/zerver/lib/bulk_create.py +++ b/zerver/lib/bulk_create.py @@ -1,13 +1,12 @@ -from django.db.models import Model - from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from django.db.models import Model + +from zerver.lib.create_user import create_user_profile, get_display_email_address from zerver.lib.initial_password import initial_password -from zerver.models import Realm, Stream, UserProfile, \ - Subscription, Recipient, RealmAuditLog -from zerver.lib.create_user import create_user_profile, \ - get_display_email_address from zerver.lib.streams import render_stream_description +from zerver.models import Realm, RealmAuditLog, Recipient, Stream, Subscription, UserProfile + def bulk_create_users(realm: Realm, users_raw: Set[Tuple[str, str, str, bool]], diff --git a/zerver/lib/cache.py b/zerver/lib/cache.py index facb9056fc..ad0a7bb320 100644 --- a/zerver/lib/cache.py +++ b/zerver/lib/cache.py @@ -1,32 +1,41 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs -from functools import wraps - -from django.utils.lru_cache import lru_cache -from django.core.cache import cache as djcache -from django.core.cache import caches -from django.conf import settings -from django.db.models import Q -from django.core.cache.backends.base import BaseCache -from django.http import HttpRequest - -from typing import Any, Callable, Dict, Iterable, List, \ - Optional, Sequence, TypeVar, Tuple, TYPE_CHECKING - -from zerver.lib.utils import statsd, statsd_key, make_safe_digest -import time import base64 +import hashlib import logging +import os import random import re import sys +import time import traceback -import os -import hashlib +from functools import wraps +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, +) + +from django.conf import settings +from django.core.cache import cache as djcache +from django.core.cache import caches +from django.core.cache.backends.base import BaseCache +from django.db.models import Q +from django.http import HttpRequest +from django.utils.lru_cache import lru_cache + +from zerver.lib.utils import make_safe_digest, statsd, statsd_key if TYPE_CHECKING: # These modules have to be imported for type annotations but # they cannot be imported at runtime due to cyclic dependency. - from zerver.models import UserProfile, Realm, Message + from zerver.models import Message, Realm, UserProfile MEMCACHED_MAX_KEY_LENGTH = 250 diff --git a/zerver/lib/cache_helpers.py b/zerver/lib/cache_helpers.py index 944b5c4c27..fe3526c25c 100644 --- a/zerver/lib/cache_helpers.py +++ b/zerver/lib/cache_helpers.py @@ -1,28 +1,38 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/caching.html for docs - -from typing import Any, Callable, Dict, List, Tuple - import datetime import logging +from importlib import import_module +from typing import Any, Callable, Dict, List, Tuple + +from django.conf import settings +from django.contrib.sessions.models import Session +from django.db.models import Q +from django.utils.timezone import now as timezone_now # This file needs to be different from cache.py because cache.py # cannot import anything from zerver.models or we'd have an import # loop from analytics.models import RealmCount -from django.conf import settings -from zerver.models import Message, UserProfile, Stream, get_stream_cache_key, \ - Client, get_client_cache_key, \ - Huddle, huddle_hash_cache_key -from zerver.lib.cache import \ - user_profile_by_api_key_cache_key, \ - user_profile_cache_key, get_remote_cache_time, get_remote_cache_requests, \ - cache_set_many, to_dict_cache_key_id +from zerver.lib.cache import ( + cache_set_many, + get_remote_cache_requests, + get_remote_cache_time, + to_dict_cache_key_id, + user_profile_by_api_key_cache_key, + user_profile_cache_key, +) from zerver.lib.message import MessageDict from zerver.lib.users import get_all_api_keys -from importlib import import_module -from django.contrib.sessions.models import Session -from django.db.models import Q -from django.utils.timezone import now as timezone_now +from zerver.models import ( + Client, + Huddle, + Message, + Stream, + UserProfile, + get_client_cache_key, + get_stream_cache_key, + huddle_hash_cache_key, +) MESSAGE_CACHE_SIZE = 75000 diff --git a/zerver/lib/camo.py b/zerver/lib/camo.py index d1c7a66f1c..5b57daa120 100644 --- a/zerver/lib/camo.py +++ b/zerver/lib/camo.py @@ -1,8 +1,10 @@ -from django.conf import settings import binascii import hashlib import hmac +from django.conf import settings + + def generate_camo_url(url: str) -> str: encoded_url = url.encode("utf-8") encoded_camo_key = settings.CAMO_KEY.encode("utf-8") diff --git a/zerver/lib/ccache.py b/zerver/lib/ccache.py index c3c746dec0..a2f159c3cf 100644 --- a/zerver/lib/ccache.py +++ b/zerver/lib/ccache.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, List, Optional +import base64 +import struct +from typing import Any, Dict, List, Optional, Union # This file is adapted from samples/shellinabox/ssh-krb-wrapper in # https://github.com/davidben/webathena, which has the following @@ -26,9 +28,6 @@ from typing import Any, Dict, List, Optional # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import base64 -import struct -from typing import Union def force_bytes(s: Union[str, bytes], encoding: str='utf-8') -> bytes: """converts a string to binary string""" diff --git a/zerver/lib/context_managers.py b/zerver/lib/context_managers.py index 1a6700726c..0ca6c81af6 100644 --- a/zerver/lib/context_managers.py +++ b/zerver/lib/context_managers.py @@ -1,11 +1,10 @@ """ Context managers, i.e. things you can use with the 'with' statement. """ - - import fcntl from contextlib import contextmanager -from typing import Iterator, IO, Any, Union +from typing import IO, Any, Iterator, Union + @contextmanager def flock(lockfile: Union[int, IO[Any]], shared: bool=False) -> Iterator[None]: diff --git a/zerver/lib/create_user.py b/zerver/lib/create_user.py index 7924db0bbb..dc8e16a771 100644 --- a/zerver/lib/create_user.py +++ b/zerver/lib/create_user.py @@ -1,14 +1,22 @@ -from django.contrib.auth.models import UserManager -from django.utils.timezone import now as timezone_now -from zerver.models import UserProfile, Recipient, Subscription, Realm, Stream, \ - PreregistrationUser, get_fake_email_domain -from zerver.lib.upload import copy_avatar -from zerver.lib.hotspots import copy_hotpots -from zerver.lib.utils import generate_api_key +from typing import Optional import ujson +from django.contrib.auth.models import UserManager +from django.utils.timezone import now as timezone_now + +from zerver.lib.hotspots import copy_hotpots +from zerver.lib.upload import copy_avatar +from zerver.lib.utils import generate_api_key +from zerver.models import ( + PreregistrationUser, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + get_fake_email_domain, +) -from typing import Optional def copy_user_settings(source_profile: UserProfile, target_profile: UserProfile) -> None: """Warning: Does not save, to avoid extra database queries""" diff --git a/zerver/lib/db.py b/zerver/lib/db.py index 2971ac2edc..32d3d89d74 100644 --- a/zerver/lib/db.py +++ b/zerver/lib/db.py @@ -1,9 +1,8 @@ import time -from psycopg2.extensions import cursor, connection -from psycopg2.sql import Composable +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, TypeVar, Union -from typing import Callable, Optional, Iterable, Any, Dict, List, Union, TypeVar, \ - Mapping, Sequence +from psycopg2.extensions import connection, cursor +from psycopg2.sql import Composable CursorObj = TypeVar('CursorObj', bound=cursor) Query = Union[str, Composable] diff --git a/zerver/lib/debug.py b/zerver/lib/debug.py index d248816db1..3b44bd5261 100644 --- a/zerver/lib/debug.py +++ b/zerver/lib/debug.py @@ -8,10 +8,10 @@ import threading import traceback import tracemalloc from types import FrameType +from typing import Optional from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import Optional logger = logging.getLogger('zulip.debug') diff --git a/zerver/lib/dev_ldap_directory.py b/zerver/lib/dev_ldap_directory.py index 2b29cab5e8..f703cad52d 100644 --- a/zerver/lib/dev_ldap_directory.py +++ b/zerver/lib/dev_ldap_directory.py @@ -4,6 +4,7 @@ import os from typing import Any, Dict, List, Optional from django.conf import settings + from zerver.lib.storage import static_path # See https://jackstromberg.com/2013/01/useraccountcontrol-attributeflag-values/ @@ -59,6 +60,7 @@ def init_fakeldap(directory: Optional[Dict[str, Dict[str, List[str]]]]=None) -> # because its dependency pbr uses pkgresources, which is # really slow to import.) from unittest import mock + from fakeldap import MockLDAP # Silent `django_auth_ldap` logger in dev mode to avoid diff --git a/zerver/lib/digest.py b/zerver/lib/digest.py index 4d7efe4915..feadb985de 100644 --- a/zerver/lib/digest.py +++ b/zerver/lib/digest.py @@ -1,21 +1,29 @@ -from typing import Any, Dict, List, Set, Tuple, Union - -from collections import defaultdict import datetime import logging +from collections import defaultdict +from typing import Any, Dict, List, Set, Tuple, Union from django.conf import settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link -from zerver.lib.email_notifications import build_message_list -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import encode_stream -from zerver.models import UserProfile, Recipient, Subscription, UserActivity, \ - get_active_streams, get_user_profile_by_id, Realm, Message, RealmAuditLog from zerver.context_processors import common_context -from zerver.lib.queue import queue_json_publish +from zerver.lib.email_notifications import build_message_list from zerver.lib.logging_util import log_to_file +from zerver.lib.queue import queue_json_publish +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.url_encoding import encode_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserProfile, + get_active_streams, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) log_to_file(logger, settings.DIGEST_LOG_PATH) diff --git a/zerver/lib/display_recipient.py b/zerver/lib/display_recipient.py index cea7d78708..2b933245cd 100644 --- a/zerver/lib/display_recipient.py +++ b/zerver/lib/display_recipient.py @@ -1,9 +1,14 @@ from typing import Dict, List, Optional, Set, Tuple -from typing_extensions import TypedDict -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.cache import cache_with_key, display_recipient_cache_key, generic_bulk_cached_fetch, \ - display_recipient_bulk_get_users_by_id_cache_key +from typing_extensions import TypedDict + +from zerver.lib.cache import ( + cache_with_key, + display_recipient_bulk_get_users_by_id_cache_key, + display_recipient_cache_key, + generic_bulk_cached_fetch, +) +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.models import Recipient, Stream, UserProfile, bulk_get_huddle_user_ids display_recipient_fields = [ diff --git a/zerver/lib/domains.py b/zerver/lib/domains.py index 307dc7d4eb..570d886efd 100644 --- a/zerver/lib/domains.py +++ b/zerver/lib/domains.py @@ -1,9 +1,9 @@ +import re +from typing import Optional + from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from typing import Optional - -import re def validate_domain(domain: Optional[str]) -> None: if domain is None or len(domain) == 0: diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index 59ef84b214..6e1ddd6d56 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -1,31 +1,44 @@ -from typing import Dict, Optional, Tuple, List - import logging import re - from email.header import decode_header, make_header from email.message import Message as EmailMessage from email.utils import getaddresses +from typing import Dict, List, Optional, Tuple from django.conf import settings -from django.utils.timezone import timedelta, now as timezone_now +from django.utils.timezone import now as timezone_now +from django.utils.timezone import timedelta -from zerver.lib.actions import internal_send_private_message, \ - internal_send_stream_message, internal_send_huddle_message -from zerver.lib.email_mirror_helpers import decode_email_address, \ - get_email_gateway_message_string_from_address, ZulipEmailForwardError +from zerver.lib.actions import ( + internal_send_huddle_message, + internal_send_private_message, + internal_send_stream_message, +) +from zerver.lib.email_mirror_helpers import ( + ZulipEmailForwardError, + decode_email_address, + get_email_gateway_message_string_from_address, +) from zerver.lib.email_notifications import convert_html_to_markdown -from zerver.lib.queue import queue_json_publish -from zerver.lib.utils import generate_random_token -from zerver.lib.upload import upload_message_file -from zerver.lib.send_email import FromAddress -from zerver.lib.rate_limiter import RateLimitedObject from zerver.lib.exceptions import RateLimited from zerver.lib.message import truncate_body, truncate_topic -from zerver.models import Stream, Recipient, MissedMessageEmailAddress, \ - get_display_recipient, \ - Message, Realm, UserProfile, get_system_bot, get_user, get_stream_by_id_in_realm - +from zerver.lib.queue import queue_json_publish +from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.send_email import FromAddress +from zerver.lib.upload import upload_message_file +from zerver.lib.utils import generate_random_token +from zerver.models import ( + Message, + MissedMessageEmailAddress, + Realm, + Recipient, + Stream, + UserProfile, + get_display_recipient, + get_stream_by_id_in_realm, + get_system_bot, + get_user, +) from zproject.backends import is_user_active logger = logging.getLogger(__name__) diff --git a/zerver/lib/email_mirror_helpers.py b/zerver/lib/email_mirror_helpers.py index f79ae02ac8..59ed695385 100644 --- a/zerver/lib/email_mirror_helpers.py +++ b/zerver/lib/email_mirror_helpers.py @@ -1,11 +1,11 @@ import re +from typing import Any, Callable, Dict, Tuple from django.conf import settings from django.utils.text import slugify from zerver.models import Stream -from typing import Any, Callable, Dict, Tuple def default_option_handler_factory(address_option: str) -> Callable[[Dict[str, Any]], None]: def option_setter(options_dict: Dict[str, Any]) -> None: diff --git a/zerver/lib/email_notifications.py b/zerver/lib/email_notifications.py index d7d6840787..9ba1b0d9ff 100644 --- a/zerver/lib/email_notifications.py +++ b/zerver/lib/email_notifications.py @@ -1,40 +1,44 @@ +import re +from collections import defaultdict +from datetime import timedelta +from email.utils import formataddr from typing import Any, Dict, Iterable, List, Optional, Tuple -from zerver.lib.types import DisplayRecipientT -from confirmation.models import one_click_unsubscribe_link +import html2text +import lxml.html +import pytz +from bs4 import BeautifulSoup from django.conf import settings +from django.contrib.auth import get_backends from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.contrib.auth import get_backends +from lxml.cssselect import CSSSelector +from confirmation.models import one_click_unsubscribe_link from zerver.decorator import statsd_increment from zerver.lib.message import bulk_access_messages from zerver.lib.queue import queue_json_publish -from zerver.lib.send_email import send_future_email, FromAddress -from zerver.lib.url_encoding import personal_narrow_url, huddle_narrow_url, \ - stream_narrow_url, topic_narrow_url +from zerver.lib.send_email import FromAddress, send_future_email +from zerver.lib.types import DisplayRecipientT +from zerver.lib.url_encoding import ( + huddle_narrow_url, + personal_narrow_url, + stream_narrow_url, + topic_narrow_url, +) from zerver.models import ( + Message, Recipient, - UserMessage, Stream, - get_display_recipient, + UserMessage, UserProfile, + get_context_for_message, + get_display_recipient, get_user_profile_by_id, receives_offline_email_notifications, - get_context_for_message, - Message, ) -from datetime import timedelta -from email.utils import formataddr -import html2text -from lxml.cssselect import CSSSelector -import lxml.html -import re -from collections import defaultdict -import pytz -from bs4 import BeautifulSoup def relative_to_full_url(base_url: str, content: str) -> str: # Convert relative URLs to absolute URLs. @@ -297,6 +301,7 @@ def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile, for a group of messages that share a recipient (and topic) """ from zerver.context_processors import common_context + # Disabled missedmessage emails internally if not user_profile.enable_offline_email_notifications: return @@ -558,7 +563,7 @@ def enqueue_welcome_emails(user: UserProfile, realm_creation: bool=False) -> Non context['getting_started_link'] = "https://zulip.com" # Imported here to avoid import cycles. - from zproject.backends import email_belongs_to_ldap, ZulipLDAPAuthBackend + from zproject.backends import ZulipLDAPAuthBackend, email_belongs_to_ldap if email_belongs_to_ldap(user.realm, user.delivery_email): context["ldap"] = True diff --git a/zerver/lib/email_validation.py b/zerver/lib/email_validation.py index d67398fd98..dc37305518 100644 --- a/zerver/lib/email_validation.py +++ b/zerver/lib/email_validation.py @@ -3,21 +3,22 @@ from typing import Callable, Dict, Optional, Set, Tuple from django.core import validators from django.core.exceptions import ValidationError from django.utils.translation import ugettext as _ -from zerver.lib.name_restrictions import is_disposable_domain +from zerver.lib.name_restrictions import is_disposable_domain # TODO: Move DisposableEmailError, etc. into here. from zerver.models import ( - email_to_username, - email_to_domain, - get_users_by_delivery_email, - is_cross_realm_bot_email, DisposableEmailError, DomainNotAllowedForRealmError, EmailContainsPlusError, Realm, RealmDomain, + email_to_domain, + email_to_username, + get_users_by_delivery_email, + is_cross_realm_bot_email, ) + def validate_disposable(email: str) -> None: if is_disposable_domain(email_to_domain(email)): raise DisposableEmailError diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index 902732d747..df339df2f2 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -1,14 +1,14 @@ import os import re -import ujson - -from django.utils.translation import ugettext as _ from typing import Optional, Tuple +import ujson +from django.utils.translation import ugettext as _ + +from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend -from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.models import Reaction, Realm, RealmEmoji, UserProfile emoji_codes_path = static_path("generated/emoji/emoji_codes.json") diff --git a/zerver/lib/error_notify.py b/zerver/lib/error_notify.py index fb3d5ddba5..21b8f09960 100644 --- a/zerver/lib/error_notify.py +++ b/zerver/lib/error_notify.py @@ -1,16 +1,17 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - from collections import defaultdict +from typing import Any, Dict, Optional + from django.conf import settings from django.core.mail import mail_admins from django.http import HttpResponse from django.utils.translation import ugettext as _ -from typing import Any, Dict, Optional from zerver.filters import clean_data_from_query_parameters -from zerver.models import get_stream, get_system_bot from zerver.lib.actions import internal_send_stream_message -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success +from zerver.models import get_stream, get_system_bot + def format_email_subject(email_subject: str) -> str: """ diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 1522dd9fcd..018b18f55d 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -1,20 +1,28 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. - import copy - -from django.utils.translation import ugettext as _ -from django.conf import settings from importlib import import_module -from typing import ( - Any, Callable, Dict, Iterable, Optional, Sequence, Set, -) +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Set + +from django.conf import settings +from django.utils.translation import ugettext as _ session_engine = import_module(settings.SESSION_ENGINE) +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.lib.actions import ( + default_stream_groups_to_dicts_sorted, + do_get_streams, + gather_subscriptions_helper, + get_available_notification_sounds, + get_default_streams_for_realm, + get_owned_bot_dicts, + streams_to_dicts_sorted, +) from zerver.lib.alert_words import user_alert_words from zerver.lib.avatar import avatar_url from zerver.lib.bot_config import load_bot_config_template +from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots from zerver.lib.integrations import EMBEDDED_BOTS, WEBHOOK_INTEGRATIONS from zerver.lib.message import ( @@ -27,40 +35,34 @@ from zerver.lib.message import ( remove_message_id_from_unread_mgs, ) from zerver.lib.narrow import check_supported_events_narrow_filter, read_stop_words -from zerver.lib.presence import ( - get_presences_for_realm, - get_presence_for_user, -) +from zerver.lib.presence import get_presence_for_user, get_presences_for_realm from zerver.lib.push_notifications import push_notifications_enabled -from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url from zerver.lib.request import JsonableError +from zerver.lib.soft_deactivation import reactivate_user_if_soft_deactivated from zerver.lib.stream_subscription import handle_stream_notifications_compatibility from zerver.lib.topic import TOPIC_NAME from zerver.lib.topic_mutes import get_topic_mutes -from zerver.lib.actions import ( - do_get_streams, - get_default_streams_for_realm, - gather_subscriptions_helper, - streams_to_dicts_sorted, - default_stream_groups_to_dicts_sorted, - get_owned_bot_dicts, - get_available_notification_sounds, -) -from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.lib.user_groups import user_groups_in_realm_serialized from zerver.lib.user_status import get_user_info_dict -from zerver.tornado.event_queue import request_event_queue, get_user_events +from zerver.lib.users import get_cross_realm_dicts, get_raw_user_data, is_administrator_role from zerver.models import ( - Client, Message, Realm, UserProfile, UserMessage, + Client, + CustomProfileField, + Message, + Realm, + Stream, + UserMessage, + UserProfile, + custom_profile_fields_for_realm, + get_default_stream_groups, + get_realm_domains, realm_filters_for_realm, - custom_profile_fields_for_realm, get_realm_domains, - get_default_stream_groups, CustomProfileField, Stream, ) +from zerver.tornado.event_queue import get_user_events, request_event_queue from zproject.backends import email_auth_enabled, password_auth_enabled -from version import ZULIP_VERSION, API_FEATURE_LEVEL -from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS + def add_realm_logo_fields(state: Dict[str, Any], realm: Realm) -> None: state['realm_logo_url'] = get_realm_logo_url(realm, night = False) diff --git a/zerver/lib/exceptions.py b/zerver/lib/exceptions.py index 897c2dca6e..57db95d943 100644 --- a/zerver/lib/exceptions.py +++ b/zerver/lib/exceptions.py @@ -1,9 +1,8 @@ from enum import Enum -from typing import Any, Dict, List, NoReturn, Type, TypeVar, Optional +from typing import Any, Dict, List, NoReturn, Optional, Type, TypeVar from django.utils.translation import ugettext as _ - T = TypeVar("T", bound="AbstractEnum") class AbstractEnum(Enum): diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 7bbe837770..090a40187f 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -7,35 +7,61 @@ # (2) if it doesn't belong in EXCLUDED_TABLES, add a Config object for # it to get_realm_config. import datetime +import glob +import logging +import os +import shutil +import subprocess +import tempfile +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union + import boto3 +import ujson from boto3.resources.base import ServiceResource from django.apps import apps from django.conf import settings from django.forms.models import model_to_dict -from django.utils.timezone import make_aware as timezone_make_aware from django.utils.timezone import is_naive as timezone_is_naive -import glob -import logging -import os -import ujson -import subprocess -import tempfile -import shutil -from scripts.lib.zulip_tools import overwrite_symlink -from zerver.lib.avatar_hash import user_avatar_path_from_ids -from analytics.models import RealmCount, UserCount, StreamCount -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, RealmFilter, Reaction, \ - RealmDomain, Recipient, DefaultStream, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, CustomProfileField, \ - CustomProfileFieldValue, get_display_recipient, Attachment, get_system_bot, \ - RealmAuditLog, UserHotspot, MutedTopic, Service, UserGroup, \ - UserGroupMembership, BotStorageData, BotConfigData -from zerver.lib.pysa import mark_sanitized +from django.utils.timezone import make_aware as timezone_make_aware import zerver.lib.upload -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Union +from analytics.models import RealmCount, StreamCount, UserCount +from scripts.lib.zulip_tools import overwrite_symlink +from zerver.lib.avatar_hash import user_avatar_path_from_ids +from zerver.lib.pysa import mark_sanitized +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + get_display_recipient, + get_system_bot, + get_user_profile_by_id, +) # Custom mypy types follow: Record = Dict[str, Any] diff --git a/zerver/lib/external_accounts.py b/zerver/lib/external_accounts.py index bcb08c17c6..81a0772895 100644 --- a/zerver/lib/external_accounts.py +++ b/zerver/lib/external_accounts.py @@ -2,11 +2,15 @@ This module stores data for "External Account" custom profile field. """ from typing import Optional + from django.utils.translation import ugettext as _ -from zerver.lib.validator import check_required_string, \ - check_external_account_url_pattern, check_dict_only from zerver.lib.types import ProfileFieldData +from zerver.lib.validator import ( + check_dict_only, + check_external_account_url_pattern, + check_required_string, +) # Default external account fields are by default available # to realm admins, where realm admin only need to select diff --git a/zerver/lib/fix_unreads.py b/zerver/lib/fix_unreads.py index 9bc46a8f4d..f13b2811f7 100644 --- a/zerver/lib/fix_unreads.py +++ b/zerver/lib/fix_unreads.py @@ -1,7 +1,7 @@ -import time import logging - +import time from typing import Callable, List, TypeVar + from psycopg2.extensions import cursor from psycopg2.sql import SQL diff --git a/zerver/lib/generate_test_data.py b/zerver/lib/generate_test_data.py index 979bd5ad82..a1432535ab 100644 --- a/zerver/lib/generate_test_data.py +++ b/zerver/lib/generate_test_data.py @@ -1,11 +1,13 @@ import itertools -import ujson -import random -from typing import List, Dict, Any import os +import random +from typing import Any, Dict, List + +import ujson from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path + def load_config() -> Dict[str, Any]: with open("zerver/tests/fixtures/config.generate_data.json") as infile: config = ujson.load(infile) diff --git a/zerver/lib/hotspots.py b/zerver/lib/hotspots.py index c37491f3d3..322654d522 100644 --- a/zerver/lib/hotspots.py +++ b/zerver/lib/hotspots.py @@ -1,11 +1,11 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/hotspots.html # for documentation on this subsystem. +from typing import Dict, List + from django.conf import settings from django.utils.translation import ugettext as _ -from zerver.models import UserProfile, UserHotspot - -from typing import List, Dict +from zerver.models import UserHotspot, UserProfile ALL_HOTSPOTS: Dict[str, Dict[str, str]] = { 'intro_reply': { diff --git a/zerver/lib/html_diff.py b/zerver/lib/html_diff.py index 16686a69e4..cc8d4fad6c 100644 --- a/zerver/lib/html_diff.py +++ b/zerver/lib/html_diff.py @@ -1,8 +1,9 @@ -import lxml - -from lxml.html.diff import htmldiff from typing import Optional +import lxml +from lxml.html.diff import htmldiff + + def highlight_with_class(text: str, klass: str) -> str: return f'{text}' diff --git a/zerver/lib/html_to_text.py b/zerver/lib/html_to_text.py index c7bac8e275..9a450f5dc8 100644 --- a/zerver/lib/html_to_text.py +++ b/zerver/lib/html_to_text.py @@ -6,6 +6,7 @@ from django.utils.html import escape from zerver.lib.cache import cache_with_key, open_graph_description_cache_key + def html_to_text(content: str, tags: Optional[Dict[str, str]]=None) -> str: bs = BeautifulSoup(content, features='lxml') # Skip any admonition (warning) blocks, since they're diff --git a/zerver/lib/i18n.py b/zerver/lib/i18n.py index b63d4c5934..8caf89ad9c 100644 --- a/zerver/lib/i18n.py +++ b/zerver/lib/i18n.py @@ -1,16 +1,15 @@ +import logging import operator +import os +from itertools import zip_longest +from typing import Any, Dict, List +import ujson from django.conf import settings from django.utils import translation -from django.utils.translation import ugettext as _ from django.utils.lru_cache import lru_cache +from django.utils.translation import ugettext as _ -from itertools import zip_longest -from typing import Any, List, Dict - -import os -import ujson -import logging def with_language(string: str, language: str) -> str: """ diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index c58673002a..ce0f769afb 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -1,44 +1,71 @@ import datetime import logging import os -import ujson import shutil +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast import boto3 +import ujson from bs4 import BeautifulSoup from django.conf import settings from django.db import connection from django.db.models import Max from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Optional, Set, Tuple, \ - Iterable, cast from psycopg2.extras import execute_values -from psycopg2.sql import Identifier, SQL +from psycopg2.sql import SQL, Identifier from analytics.models import RealmCount, StreamCount, UserCount -from zerver.lib.actions import UserMessageLite, bulk_insert_ums, \ - do_change_plan_type, do_change_avatar_fields +from zerver.lib.actions import ( + UserMessageLite, + bulk_insert_ums, + do_change_avatar_fields, + do_change_plan_type, +) from zerver.lib.avatar_hash import user_avatar_path_from_ids -from zerver.lib.bulk_create import bulk_create_users, bulk_set_users_or_streams_recipient_fields -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.export import DATE_FIELDS, \ - Record, TableData, TableName, Field, Path -from zerver.lib.message import do_render_markdown from zerver.lib.bugdown import version as bugdown_version -from zerver.lib.streams import render_stream_description -from zerver.lib.upload import random_name, sanitize_name, \ - guess_type, BadImageError -from zerver.lib.utils import generate_api_key, process_list_in_batches +from zerver.lib.bulk_create import bulk_create_users, bulk_set_users_or_streams_recipient_fields +from zerver.lib.export import DATE_FIELDS, Field, Path, Record, TableData, TableName +from zerver.lib.message import do_render_markdown from zerver.lib.parallel import run_parallel -from zerver.lib.server_initialization import server_initialized, create_internal_realm -from zerver.models import UserProfile, Realm, Client, Huddle, Stream, \ - UserMessage, Subscription, Message, RealmEmoji, \ - RealmDomain, Recipient, get_user_profile_by_id, \ - UserPresence, UserActivity, UserActivityInterval, Reaction, \ - CustomProfileField, CustomProfileFieldValue, RealmAuditLog, \ - Attachment, get_system_bot, email_to_username, get_huddle_hash, \ - UserHotspot, MutedTopic, Service, UserGroup, UserGroupMembership, \ - BotStorageData, BotConfigData, DefaultStream, RealmFilter +from zerver.lib.server_initialization import create_internal_realm, server_initialized +from zerver.lib.streams import render_stream_description +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.upload import BadImageError, guess_type, random_name, sanitize_name +from zerver.lib.utils import generate_api_key, process_list_in_batches +from zerver.models import ( + Attachment, + BotConfigData, + BotStorageData, + Client, + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Huddle, + Message, + MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmEmoji, + RealmFilter, + Recipient, + Service, + Stream, + Subscription, + UserActivity, + UserActivityInterval, + UserGroup, + UserGroupMembership, + UserHotspot, + UserMessage, + UserPresence, + UserProfile, + email_to_username, + get_huddle_hash, + get_system_bot, + get_user_profile_by_id, +) realm_tables = [("zerver_defaultstream", DefaultStream, "defaultstream"), ("zerver_realmemoji", RealmEmoji, "realmemoji"), @@ -707,6 +734,7 @@ def import_uploads(realm: Realm, import_dir: Path, processes: int, processing_av if processing_avatars: from zerver.lib.upload import upload_backend + # Ensure that we have medium-size avatar images for every # avatar. TODO: This implementation is hacky, both in that it # does get_user_profile_by_id for each user, and in that it diff --git a/zerver/lib/initial_password.py b/zerver/lib/initial_password.py index 4e2d5360f9..e53a088ef1 100644 --- a/zerver/lib/initial_password.py +++ b/zerver/lib/initial_password.py @@ -1,10 +1,9 @@ -from django.conf import settings - -import hashlib import base64 - +import hashlib from typing import Optional +from django.conf import settings + def initial_password(email: str) -> Optional[str]: """Given an email address, returns the initial password for that account, as diff --git a/zerver/lib/integrations.py b/zerver/lib/integrations.py index abadb2db15..8a1c81e1fd 100644 --- a/zerver/lib/integrations.py +++ b/zerver/lib/integrations.py @@ -1,16 +1,16 @@ import os - from dataclasses import dataclass -from typing import Dict, List, Optional, Any, Tuple +from typing import Any, Dict, List, Optional, Tuple + from django.conf.urls import url from django.contrib.staticfiles.storage import staticfiles_storage from django.urls.resolvers import RegexPattern from django.utils.module_loading import import_string from django.utils.translation import ugettext as _ + from zerver.lib.storage import static_path from zerver.lib.types import Validator - """This module declares all of the (documented) integrations available in the Zulip server. The Integration class is used as part of generating the documentation on the /integrations page, while the diff --git a/zerver/lib/logging_util.py b/zerver/lib/logging_util.py index 23ddb43e21..e250494fd9 100644 --- a/zerver/lib/logging_util.py +++ b/zerver/lib/logging_util.py @@ -1,16 +1,16 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - -from django.utils.timezone import now as timezone_now - import hashlib import logging import threading import traceback -from typing import Optional, Tuple from datetime import datetime, timedelta, timezone +from logging import Logger +from typing import Optional, Tuple + from django.conf import settings from django.core.cache import cache -from logging import Logger +from django.utils.timezone import now as timezone_now + class _RateLimitFilter: """This class is designed to rate-limit Django error reporting diff --git a/zerver/lib/management.py b/zerver/lib/management.py index f7c90fa9f5..3da9b47a6e 100644 --- a/zerver/lib/management.py +++ b/zerver/lib/management.py @@ -1,15 +1,14 @@ # Library code for use in management commands - import time - from argparse import ArgumentParser, RawTextHelpFormatter +from typing import Any, Dict, List, Optional from django.conf import settings from django.core.exceptions import MultipleObjectsReturned from django.core.management.base import BaseCommand, CommandError -from typing import Any, Dict, Optional, List -from zerver.models import Realm, UserProfile, Client, get_client +from zerver.models import Client, Realm, UserProfile, get_client + def is_integer_string(val: str) -> bool: try: diff --git a/zerver/lib/mdiff.py b/zerver/lib/mdiff.py index 09a2a93098..3bde46030f 100755 --- a/zerver/lib/mdiff.py +++ b/zerver/lib/mdiff.py @@ -1,6 +1,7 @@ +import logging import os import subprocess -import logging + class DiffException(Exception): pass diff --git a/zerver/lib/mention.py b/zerver/lib/mention.py index b1fc0e8e9c..2691190dc1 100644 --- a/zerver/lib/mention.py +++ b/zerver/lib/mention.py @@ -1,6 +1,5 @@ -from typing import Optional, Set, Tuple - import re +from typing import Optional, Set, Tuple # Match multi-word string between @** ** or match any one-word # sequences after @ diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 1d783e39e7..5be36afc37 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -1,64 +1,54 @@ -import datetime -import ujson -import zlib -import ahocorasick import copy +import datetime +import zlib +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import ahocorasick +import ujson from django.db import connection from django.db.models import Sum +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ from psycopg2.sql import SQL +from typing_extensions import TypedDict from analytics.lib.counts import COUNT_STATS, RealmCount - +from zerver.lib import bugdown as bugdown from zerver.lib.avatar import get_avatar_field -import zerver.lib.bugdown as bugdown from zerver.lib.cache import ( cache_with_key, generic_bulk_cached_fetch, to_dict_cache_key, to_dict_cache_key_id, ) -from zerver.lib.display_recipient import UserDisplayRecipient, DisplayRecipientT, \ - bulk_fetch_display_recipients +from zerver.lib.display_recipient import ( + DisplayRecipientT, + UserDisplayRecipient, + bulk_fetch_display_recipients, +) from zerver.lib.request import JsonableError -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, -) +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - DB_TOPIC_NAME, - MESSAGE__TOPIC, - TOPIC_LINKS, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - build_topic_mute_checker, - topic_is_muted, -) - +from zerver.lib.topic import DB_TOPIC_NAME, MESSAGE__TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import build_topic_mute_checker, topic_is_muted from zerver.models import ( - get_display_recipient_by_id, - get_user_profile_by_id, - query_for_ids, + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, Message, + Reaction, Realm, Recipient, Stream, SubMessage, Subscription, - UserProfile, UserMessage, - Reaction, + UserProfile, + get_display_recipient_by_id, + get_user_profile_by_id, get_usermessage_by_message_id, - MAX_MESSAGE_LENGTH, - MAX_TOPIC_NAME_LENGTH, + query_for_ids, ) -from typing import Any, Dict, List, Optional, Set, Tuple, Sequence -from typing_extensions import TypedDict - RealmAlertWord = Dict[int, List[str]] class RawUnreadMessagesResult(TypedDict): diff --git a/zerver/lib/migrate.py b/zerver/lib/migrate.py index 3fa2bd3167..8fb5399261 100644 --- a/zerver/lib/migrate.py +++ b/zerver/lib/migrate.py @@ -1,8 +1,8 @@ -from psycopg2.extensions import cursor -from psycopg2.sql import Composable, Identifier, SQL +import time from typing import List, TypeVar -import time +from psycopg2.extensions import cursor +from psycopg2.sql import SQL, Composable, Identifier CursorObj = TypeVar('CursorObj', bound=cursor) diff --git a/zerver/lib/mobile_auth_otp.py b/zerver/lib/mobile_auth_otp.py index 83df1fef71..bd009fc92e 100644 --- a/zerver/lib/mobile_auth_otp.py +++ b/zerver/lib/mobile_auth_otp.py @@ -7,10 +7,11 @@ # # The decryption logic here isn't actually used by the flow; we just # have it here as part of testing the overall library. - import binascii + from zerver.models import UserProfile + def xor_hex_strings(bytes_a: str, bytes_b: str) -> str: """Given two hex strings of equal length, return a hex string with the bitwise xor of the two hex strings.""" diff --git a/zerver/lib/name_restrictions.py b/zerver/lib/name_restrictions.py index 2db6d57ceb..02f3f9f401 100644 --- a/zerver/lib/name_restrictions.py +++ b/zerver/lib/name_restrictions.py @@ -1,5 +1,6 @@ from disposable_email_domains import blacklist + def is_reserved_subdomain(subdomain: str) -> bool: if subdomain in ZULIP_RESERVED_SUBDOMAINS: return True diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.py index d3c080a8e5..e77f189130 100644 --- a/zerver/lib/narrow.py +++ b/zerver/lib/narrow.py @@ -1,13 +1,11 @@ import os +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence -from zerver.lib.request import JsonableError -from zerver.lib.topic import ( - get_topic_from_message_info, -) from django.conf import settings from django.utils.translation import ugettext as _ -from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence +from zerver.lib.request import JsonableError +from zerver.lib.topic import get_topic_from_message_info stop_words_list: Optional[List[str]] = None def read_stop_words() -> List[str]: diff --git a/zerver/lib/onboarding.py b/zerver/lib/onboarding.py index bcdde3ac5a..9b8b593a2d 100644 --- a/zerver/lib/onboarding.py +++ b/zerver/lib/onboarding.py @@ -1,15 +1,19 @@ +from typing import Dict, List + from django.conf import settings from django.db.models import Count from django.utils.translation import ugettext as _ -from zerver.lib.actions import \ - internal_prep_stream_message_by_name, internal_send_private_message, \ - do_send_messages, \ - do_add_reaction, create_users +from zerver.lib.actions import ( + create_users, + do_add_reaction, + do_send_messages, + internal_prep_stream_message_by_name, + internal_send_private_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.models import Message, Realm, UserProfile, get_system_bot -from typing import Dict, List def missing_any_realm_internal_bots() -> bool: bot_emails = [bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN,) diff --git a/zerver/lib/outgoing_webhook.py b/zerver/lib/outgoing_webhook.py index 88f2e62255..bfcc7e2ac9 100644 --- a/zerver/lib/outgoing_webhook.py +++ b/zerver/lib/outgoing_webhook.py @@ -1,23 +1,29 @@ -from typing import Any, AnyStr, Dict, Optional - import abc -import requests import json import logging +from typing import Any, AnyStr, Dict, Optional + +import requests +from django.utils.translation import ugettext as _ from requests import Response -from django.utils.translation import ugettext as _ - -from zerver.models import UserProfile, get_user_profile_by_id, get_client, \ - GENERIC_INTERFACE, Service, SLACK_INTERFACE, email_to_domain +from version import ZULIP_VERSION +from zerver.decorator import JsonableError from zerver.lib.actions import check_send_message from zerver.lib.message import MessageDict from zerver.lib.queue import retry_event from zerver.lib.topic import get_topic_from_message_info from zerver.lib.url_encoding import near_message_url -from zerver.decorator import JsonableError +from zerver.models import ( + GENERIC_INTERFACE, + SLACK_INTERFACE, + Service, + UserProfile, + email_to_domain, + get_client, + get_user_profile_by_id, +) -from version import ZULIP_VERSION class OutgoingWebhookServiceInterface(metaclass=abc.ABCMeta): diff --git a/zerver/lib/parallel.py b/zerver/lib/parallel.py index 91765874bf..b182b73aae 100644 --- a/zerver/lib/parallel.py +++ b/zerver/lib/parallel.py @@ -1,9 +1,8 @@ -from typing import Dict, Iterable, Tuple, Callable, TypeVar, Iterator - +import errno import os import pty import sys -import errno +from typing import Callable, Dict, Iterable, Iterator, Tuple, TypeVar JobData = TypeVar('JobData') diff --git a/zerver/lib/presence.py b/zerver/lib/presence.py index 58c99a343d..425153f628 100644 --- a/zerver/lib/presence.py +++ b/zerver/lib/presence.py @@ -1,21 +1,14 @@ -from collections import defaultdict - import datetime import itertools import time +from collections import defaultdict +from typing import Any, Dict, List, Set from django.utils.timezone import now as timezone_now -from typing import Any, Dict, List, Set - from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - query_for_ids, - PushDeviceToken, - Realm, - UserPresence, - UserProfile, -) +from zerver.models import PushDeviceToken, Realm, UserPresence, UserProfile, query_for_ids + def get_status_dicts_for_rows(all_rows: List[Dict[str, Any]], mobile_user_ids: Set[int], diff --git a/zerver/lib/profile.py b/zerver/lib/profile.py index 200a434e5d..3aac3844be 100644 --- a/zerver/lib/profile.py +++ b/zerver/lib/profile.py @@ -1,7 +1,6 @@ import cProfile - from functools import wraps -from typing import Any, TypeVar, Callable +from typing import Any, Callable, TypeVar ReturnT = TypeVar('ReturnT') diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index 5abbdd370a..1029eab664 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -1,32 +1,37 @@ import base64 import binascii import logging -import lxml.html import re import time +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING, Union - +import gcm +import lxml.html +import ujson from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import F from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -import gcm -import ujson from zerver.decorator import statsd_increment from zerver.lib.avatar import absolute_avatar_url from zerver.lib.exceptions import JsonableError -from zerver.lib.message import access_message, \ - bulk_access_messages_expect_usermessage, huddle_users -from zerver.lib.remote_server import send_to_push_bouncer, send_json_to_push_bouncer +from zerver.lib.message import access_message, bulk_access_messages_expect_usermessage, huddle_users +from zerver.lib.remote_server import send_json_to_push_bouncer, send_to_push_bouncer from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import PushDeviceToken, Message, Recipient, \ - UserMessage, UserProfile, \ - get_display_recipient, receives_offline_push_notifications, \ - receives_online_notifications, get_user_profile_by_id, \ - ArchivedMessage +from zerver.models import ( + ArchivedMessage, + Message, + PushDeviceToken, + Recipient, + UserMessage, + UserProfile, + get_display_recipient, + get_user_profile_by_id, + receives_offline_push_notifications, + receives_online_notifications, +) if TYPE_CHECKING: from apns2.client import APNsClient diff --git a/zerver/lib/pysa.py b/zerver/lib/pysa.py index 6019b29378..c0fa12484a 100644 --- a/zerver/lib/pysa.py +++ b/zerver/lib/pysa.py @@ -1,6 +1,5 @@ from typing import TypeVar - T = TypeVar("T") diff --git a/zerver/lib/queue.py b/zerver/lib/queue.py index 176a06ac02..f132e2a11a 100644 --- a/zerver/lib/queue.py +++ b/zerver/lib/queue.py @@ -1,17 +1,17 @@ -from collections import defaultdict import logging import random import threading import time +from collections import defaultdict from typing import Any, Callable, Dict, List, Mapping, Optional, Set -from django.conf import settings import pika import pika.adapters.tornado_connection +import ujson +from django.conf import settings from pika.adapters.blocking_connection import BlockingChannel from pika.spec import Basic from tornado import ioloop -import ujson from zerver.lib.utils import statsd diff --git a/zerver/lib/rate_limiter.py b/zerver/lib/rate_limiter.py index 87c7fbf944..78826dc48d 100644 --- a/zerver/lib/rate_limiter.py +++ b/zerver/lib/rate_limiter.py @@ -1,20 +1,18 @@ +import logging import os - +import time from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple, Type +import redis from django.conf import settings from django.http import HttpRequest + from zerver.lib.exceptions import RateLimited from zerver.lib.redis_utils import get_redis_client from zerver.lib.utils import statsd - from zerver.models import UserProfile -import logging -import redis -import time - # Implement a rate-limiting scheme inspired by the one described here, but heavily modified # https://www.domaintools.com/resources/blog/rate-limiting-with-redis diff --git a/zerver/lib/realm_description.py b/zerver/lib/realm_description.py index 39b10e9156..409a4d3fbd 100644 --- a/zerver/lib/realm_description.py +++ b/zerver/lib/realm_description.py @@ -1,8 +1,12 @@ -from zerver.models import Realm -from zerver.lib.cache import cache_with_key, realm_rendered_description_cache_key, \ - realm_text_description_cache_key from zerver.lib.bugdown import convert as bugdown_convert +from zerver.lib.cache import ( + cache_with_key, + realm_rendered_description_cache_key, + realm_text_description_cache_key, +) from zerver.lib.html_to_text import html_to_text +from zerver.models import Realm + @cache_with_key(realm_rendered_description_cache_key, timeout=3600*24*7) def get_realm_rendered_description(realm: Realm) -> str: diff --git a/zerver/lib/realm_icon.py b/zerver/lib/realm_icon.py index 81a9eeb1d1..3813d08048 100644 --- a/zerver/lib/realm_icon.py +++ b/zerver/lib/realm_icon.py @@ -4,6 +4,7 @@ from zerver.lib.avatar_hash import gravatar_hash from zerver.lib.upload import upload_backend from zerver.models import Realm + def realm_icon_url(realm: Realm) -> str: return get_realm_icon_url(realm) diff --git a/zerver/lib/realm_logo.py b/zerver/lib/realm_logo.py index 6c3eac83ea..a4273ac805 100644 --- a/zerver/lib/realm_logo.py +++ b/zerver/lib/realm_logo.py @@ -5,6 +5,7 @@ from django.conf import settings from zerver.lib.upload import upload_backend from zerver.models import Realm + def get_realm_logo_url(realm: Realm, night: bool) -> str: if night: logo_source = realm.night_logo_source diff --git a/zerver/lib/redis_utils.py b/zerver/lib/redis_utils.py index 11144e1627..af150680bb 100644 --- a/zerver/lib/redis_utils.py +++ b/zerver/lib/redis_utils.py @@ -1,10 +1,11 @@ -from django.conf import settings -from typing import Any, Dict, Optional -from zerver.lib.utils import generate_random_token - import re +from typing import Any, Dict, Optional + import redis import ujson +from django.conf import settings + +from zerver.lib.utils import generate_random_token # Redis accepts keys up to 512MB in size, but there's no reason for us to use such size, # so we want to stay limited to 1024 characters. diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 892c249b79..7ed113fe44 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -1,9 +1,9 @@ import logging -import requests -import ujson import urllib from typing import Any, Dict, List, Optional, Tuple, Union +import requests +import ujson from django.conf import settings from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ @@ -14,6 +14,7 @@ from zerver.lib.exceptions import JsonableError from zerver.lib.export import floatify_datetime_fields from zerver.models import RealmAuditLog + class PushNotificationBouncerException(Exception): pass diff --git a/zerver/lib/request.py b/zerver/lib/request.py index 515c420998..d542445b57 100644 --- a/zerver/lib/request.py +++ b/zerver/lib/request.py @@ -1,18 +1,16 @@ from collections import defaultdict from functools import wraps from types import FunctionType +from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload + import ujson - +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from typing_extensions import Literal -from zerver.lib.exceptions import JsonableError, ErrorCode, \ - InvalidJSONError +from zerver.lib.exceptions import ErrorCode, InvalidJSONError, JsonableError from zerver.lib.types import Validator, ViewFuncT -from django.http import HttpRequest, HttpResponse - -from typing import Any, Callable, Dict, Generic, List, Optional, TypeVar, Union, cast, overload -from typing_extensions import Literal class RequestConfusingParmsError(JsonableError): code = ErrorCode.REQUEST_CONFUSING_VAR diff --git a/zerver/lib/response.py b/zerver/lib/response.py index 52e1576401..a9e5516a76 100644 --- a/zerver/lib/response.py +++ b/zerver/lib/response.py @@ -1,10 +1,12 @@ +from typing import Any, Dict, List, Optional + +import ujson from django.http import HttpResponse, HttpResponseNotAllowed from django.utils.translation import ugettext as _ -import ujson -from typing import Optional, Any, Dict, List from zerver.lib.exceptions import JsonableError + class HttpResponseUnauthorized(HttpResponse): status_code = 401 diff --git a/zerver/lib/rest.py b/zerver/lib/rest.py index 3954b4aa05..1430ac21df 100644 --- a/zerver/lib/rest.py +++ b/zerver/lib/rest.py @@ -1,16 +1,20 @@ from functools import wraps from typing import Any, Callable, Dict -from django.utils.module_loading import import_string +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.utils.cache import add_never_cache_headers +from django.utils.module_loading import import_string from django.views.decorators.csrf import csrf_exempt, csrf_protect -from zerver.decorator import authenticated_json_view, authenticated_rest_api_view, \ - process_as_post, authenticated_uploads_api_view, \ - ReturnT +from zerver.decorator import ( + ReturnT, + authenticated_json_view, + authenticated_rest_api_view, + authenticated_uploads_api_view, + process_as_post, +) from zerver.lib.response import json_method_not_allowed, json_unauthorized -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect -from django.conf import settings METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH') FLAGS = ('override_api_url_scheme') diff --git a/zerver/lib/retention.py b/zerver/lib/retention.py index f23a03319c..8320c56ac7 100644 --- a/zerver/lib/retention.py +++ b/zerver/lib/retention.py @@ -1,20 +1,30 @@ +import logging from datetime import timedelta +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings from django.db import connection, transaction from django.db.models import Model from django.utils.timezone import now as timezone_now -from psycopg2.sql import Composable, Identifier, Literal, SQL +from psycopg2.sql import SQL, Composable, Identifier, Literal from zerver.lib.logging_util import log_to_file -from zerver.models import (Message, UserMessage, ArchivedUserMessage, Realm, - Attachment, ArchivedAttachment, Reaction, ArchivedReaction, - SubMessage, ArchivedSubMessage, Recipient, Stream, ArchiveTransaction, - get_user_including_cross_realm) - -from typing import Any, Dict, List, Optional, Tuple - -import logging +from zerver.models import ( + ArchivedAttachment, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Recipient, + Stream, + SubMessage, + UserMessage, + get_user_including_cross_realm, +) logger = logging.getLogger('zulip.retention') log_to_file(logger, settings.RETENTION_LOG_PATH) diff --git a/zerver/lib/send_email.py b/zerver/lib/send_email.py index 955786c9f2..ee9a523ed9 100644 --- a/zerver/lib/send_email.py +++ b/zerver/lib/send_email.py @@ -1,30 +1,26 @@ +import datetime +import hashlib +import logging +import os +from email.parser import Parser +from email.policy import default +from email.utils import formataddr, parseaddr +from typing import Any, Dict, List, Mapping, Optional, Tuple + +import ujson from django.conf import settings from django.core.mail import EmailMultiAlternatives from django.core.management import CommandError from django.template import loader +from django.template.exceptions import TemplateDoesNotExist from django.utils.timezone import now as timezone_now from django.utils.translation import override as override_language from django.utils.translation import ugettext as _ -from django.template.exceptions import TemplateDoesNotExist -from scripts.setup.inline_email_css import inline_template -from zerver.models import ScheduledEmail, get_user_profile_by_id, \ - EMAIL_TYPES, Realm, UserProfile - -import datetime -from email.utils import parseaddr, formataddr -from email.parser import Parser -from email.policy import default - -import logging -import ujson -import hashlib - -import os -from typing import Any, Dict, List, Mapping, Optional, Tuple - -from zerver.lib.logging_util import log_to_file from confirmation.models import generate_key +from scripts.setup.inline_email_css import inline_template +from zerver.lib.logging_util import log_to_file +from zerver.models import EMAIL_TYPES, Realm, ScheduledEmail, UserProfile, get_user_profile_by_id ## Logging setup ## diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py index cf74b46cb2..cdf0185760 100644 --- a/zerver/lib/server_initialization.py +++ b/zerver/lib/server_initialization.py @@ -1,10 +1,10 @@ +from typing import Iterable, Optional, Tuple + from django.conf import settings from zerver.lib.bulk_create import bulk_create_users -from zerver.models import Realm, UserProfile, email_to_username, get_client, \ - get_system_bot +from zerver.models import Realm, UserProfile, email_to_username, get_client, get_system_bot -from typing import Iterable, Optional, Tuple def server_initialized() -> bool: return Realm.objects.exists() diff --git a/zerver/lib/sessions.py b/zerver/lib/sessions.py index d61a4cbecd..63422aadcb 100644 --- a/zerver/lib/sessions.py +++ b/zerver/lib/sessions.py @@ -1,15 +1,15 @@ import logging - from datetime import timedelta +from importlib import import_module +from typing import Any, List, Mapping, Optional + from django.conf import settings from django.contrib.auth import SESSION_KEY, get_user_model from django.contrib.sessions.models import Session from django.utils.timezone import now as timezone_now -from importlib import import_module -from typing import Any, List, Mapping, Optional -from zerver.models import Realm, UserProfile, get_user_profile_by_id from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime +from zerver.models import Realm, UserProfile, get_user_profile_by_id session_engine = import_module(settings.SESSION_ENGINE) diff --git a/zerver/lib/soft_deactivation.py b/zerver/lib/soft_deactivation.py index bd0e2d1fbe..f6f2aa45e3 100644 --- a/zerver/lib/soft_deactivation.py +++ b/zerver/lib/soft_deactivation.py @@ -1,16 +1,24 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation - -from zerver.lib.logging_util import log_to_file -from collections import defaultdict import logging +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List, Optional, Union + +from django.conf import settings from django.db import transaction from django.db.models import Max -from django.conf import settings from django.utils.timezone import now as timezone_now -from typing import DefaultDict, Dict, List, Optional, Union, Any -from zerver.models import UserProfile, UserMessage, RealmAuditLog, \ - Subscription, Message, Recipient, UserActivity, Realm +from zerver.lib.logging_util import log_to_file +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + Recipient, + Subscription, + UserActivity, + UserMessage, + UserProfile, +) logger = logging.getLogger("zulip.soft_deactivation") log_to_file(logger, settings.SOFT_DEACTIVATION_LOG_PATH) diff --git a/zerver/lib/sqlalchemy_utils.py b/zerver/lib/sqlalchemy_utils.py index 1770f70c09..c6314bbf8f 100644 --- a/zerver/lib/sqlalchemy_utils.py +++ b/zerver/lib/sqlalchemy_utils.py @@ -1,9 +1,10 @@ -from typing import Optional, Any - -from django.db import connection -from zerver.lib.db import TimeTrackingConnection +from typing import Any, Optional import sqlalchemy +from django.db import connection + +from zerver.lib.db import TimeTrackingConnection + # This is a Pool that doesn't close connections. Therefore it can be used with # existing Django database connections. diff --git a/zerver/lib/statistics.py b/zerver/lib/statistics.py index 72d4f47356..7b41508e2d 100644 --- a/zerver/lib/statistics.py +++ b/zerver/lib/statistics.py @@ -1,7 +1,8 @@ -from zerver.models import UserProfile, UserActivityInterval - from datetime import datetime, timedelta +from zerver.models import UserActivityInterval, UserProfile + + # Return the amount of Zulip usage for this user between the two # given dates def seconds_usage_between(user_profile: UserProfile, begin: datetime, end: datetime) -> timedelta: diff --git a/zerver/lib/stream_recipient.py b/zerver/lib/stream_recipient.py index d25cab5cb3..7d96d7b90b 100644 --- a/zerver/lib/stream_recipient.py +++ b/zerver/lib/stream_recipient.py @@ -1,9 +1,11 @@ -from typing import (Dict, List) +from typing import Dict, List from django.db import connection from psycopg2.sql import SQL + from zerver.models import Recipient + class StreamRecipientMap: ''' This class maps stream_id -> recipient_id and vice versa. diff --git a/zerver/lib/stream_subscription.py b/zerver/lib/stream_subscription.py index dc9b6d2c0b..2185083fbd 100644 --- a/zerver/lib/stream_subscription.py +++ b/zerver/lib/stream_subscription.py @@ -1,12 +1,9 @@ from typing import Any, Dict, List, Tuple from django.db.models.query import QuerySet -from zerver.models import ( - Recipient, - Stream, - Subscription, - UserProfile, -) + +from zerver.models import Recipient, Stream, Subscription, UserProfile + def get_active_subscriptions_for_stream_id(stream_id: int) -> QuerySet: # TODO: Change return type to QuerySet[Subscription] diff --git a/zerver/lib/stream_topic.py b/zerver/lib/stream_topic.py index d3bacf1a1e..e5fd1d4c64 100644 --- a/zerver/lib/stream_topic.py +++ b/zerver/lib/stream_topic.py @@ -1,13 +1,10 @@ from typing import Set + from django.db.models.query import QuerySet -from zerver.lib.stream_subscription import ( - get_active_subscriptions_for_stream_id, -) +from zerver.lib.stream_subscription import get_active_subscriptions_for_stream_id +from zerver.models import MutedTopic -from zerver.models import ( - MutedTopic, -) class StreamTopicTarget: ''' diff --git a/zerver/lib/streams.py b/zerver/lib/streams.py index 3f7158ddf9..cf96a5d9c5 100644 --- a/zerver/lib/streams.py +++ b/zerver/lib/streams.py @@ -1,19 +1,27 @@ -from typing import Any, Iterable, List, Mapping, Set, Tuple, Optional, Union +from typing import Any, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ from django.conf import settings +from django.db.models.query import QuerySet +from django.utils.translation import ugettext as _ +from zerver.lib.bugdown import convert as bugdown_convert from zerver.lib.request import JsonableError from zerver.models import ( - UserProfile, Stream, Subscription, Realm, Recipient, get_stream, - bulk_get_streams, get_realm_stream, DefaultStreamGroup, get_stream_by_id_in_realm, - is_cross_realm_bot_email, active_non_guest_user_ids, + DefaultStreamGroup, + Realm, + Recipient, + Stream, + Subscription, + UserProfile, + active_non_guest_user_ids, + bulk_get_streams, + get_realm_stream, + get_stream, + get_stream_by_id_in_realm, + is_cross_realm_bot_email, ) -from zerver.lib.bugdown import convert as bugdown_convert from zerver.tornado.event_queue import send_event -from django.db.models.query import QuerySet - def get_default_value_for_history_public_to_subscribers( realm: Realm, diff --git a/zerver/lib/subdomains.py b/zerver/lib/subdomains.py index 77800dc1aa..824c95937d 100644 --- a/zerver/lib/subdomains.py +++ b/zerver/lib/subdomains.py @@ -1,10 +1,12 @@ -from django.conf import settings -from django.http import HttpRequest import re from typing import Optional +from django.conf import settings +from django.http import HttpRequest + from zerver.models import Realm, UserProfile + def get_subdomain(request: HttpRequest) -> str: # The HTTP spec allows, but doesn't require, a client to omit the diff --git a/zerver/lib/test_classes.py b/zerver/lib/test_classes.py index d96ad05dcb..de686cd6de 100644 --- a/zerver/lib/test_classes.py +++ b/zerver/lib/test_classes.py @@ -1,59 +1,49 @@ +import base64 +import os +import re +import shutil +import tempfile +import urllib from contextlib import contextmanager from email.utils import parseaddr -from fakeldap import MockLDAP -from typing import (cast, Any, Dict, Iterable, - Iterator, List, Optional, - Tuple, Union, Set) +from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Tuple, Union, cast +from unittest import mock +import ujson from django.apps import apps -from django.db.migrations.state import StateApps -from django.urls import resolve from django.conf import settings -from django.test import TestCase -from django.test.client import ( - BOUNDARY, MULTIPART_CONTENT, encode_multipart, -) -from django.test.testcases import SerializeMixin -from django.http import HttpResponse -from django.db.migrations.executor import MigrationExecutor from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.db.migrations.state import StateApps from django.db.utils import IntegrityError -from django.http import HttpRequest +from django.http import HttpRequest, HttpResponse +from django.test import TestCase +from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart +from django.test.testcases import SerializeMixin +from django.urls import resolve from django.utils import translation - +from fakeldap import MockLDAP from two_factor.models import PhoneDevice -from zerver.lib.initial_password import initial_password -from zerver.lib.users import get_api_key -from zerver.lib.sessions import get_session_dict_user -from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.decorator import do_two_factor_login from zerver.lib.actions import ( - check_send_message, bulk_add_subscriptions, + bulk_add_subscriptions, bulk_remove_subscriptions, - check_send_stream_message, gather_subscriptions, + check_send_message, + check_send_stream_message, + gather_subscriptions, ) +from zerver.lib.initial_password import initial_password +from zerver.lib.sessions import get_session_dict_user +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user from zerver.lib.streams import ( create_stream_if_needed, get_default_value_for_history_public_to_subscribers, ) -from zerver.lib.stream_subscription import ( - get_stream_subscriptions_for_user, -) - -from zerver.lib.test_helpers import ( - instrument_url, find_key_by_email, -) - +from zerver.lib.test_helpers import find_key_by_email, instrument_url +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers from zerver.models import ( - clear_supported_auth_backends_cache, - flush_per_request_caches, - get_stream, - get_client, - get_display_recipient, - get_user, - get_user_by_delivery_email, - get_realm, - get_system_bot, Client, Message, Realm, @@ -61,20 +51,20 @@ from zerver.models import ( Stream, Subscription, UserProfile, + clear_supported_auth_backends_cache, + flush_per_request_caches, + get_client, + get_display_recipient, + get_realm, get_realm_stream, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, ) -from zilencer.models import get_remote_server_by_uuid -from zerver.decorator import do_two_factor_login from zerver.tornado.event_queue import clear_client_event_queues_for_testing +from zilencer.models import get_remote_server_by_uuid -import base64 -from unittest import mock -import os -import re -import ujson -import urllib -import shutil -import tempfile class UploadSerializeMixin(SerializeMixin): """ diff --git a/zerver/lib/test_fixtures.py b/zerver/lib/test_fixtures.py index b038dc627f..8d918aa0c5 100644 --- a/zerver/lib/test_fixtures.py +++ b/zerver/lib/test_fixtures.py @@ -1,27 +1,29 @@ +import glob import json import os import re +import shutil import subprocess import sys -from typing import Any, List, Set +import time from importlib import import_module from io import StringIO -import glob -import time -import shutil +from typing import Any, List, Set -from django.db import connections, DEFAULT_DB_ALIAS, ProgrammingError, \ - connection -from django.db.utils import OperationalError from django.apps import apps from django.conf import settings from django.core.management import call_command +from django.db import DEFAULT_DB_ALIAS, ProgrammingError, connection, connections +from django.db.utils import OperationalError from django.utils.module_loading import module_has_submodule sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from scripts.lib.zulip_tools import ( - get_dev_uuid_var_path, run, TEMPLATE_DATABASE_DIR, - is_digest_obsolete, write_new_digest, + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + is_digest_obsolete, + run, + write_new_digest, ) UUID_VAR_DIR = get_dev_uuid_var_path() diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index 3f0bcb302a..d1d7c9ef12 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -1,57 +1,68 @@ from contextlib import contextmanager from typing import ( - Any, Callable, Dict, Generator, Iterable, Iterator, List, Mapping, - Optional, Tuple, Union, IO, TypeVar, TYPE_CHECKING, + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union, ) -from django.urls import URLResolver -from django.conf import settings -from django.test import override_settings -from django.http import HttpResponse, HttpResponseRedirect -from django.db.migrations.state import StateApps import boto3 from boto3.resources.base import ServiceResource +from django.conf import settings +from django.db.migrations.state import StateApps +from django.http import HttpResponse, HttpResponseRedirect +from django.test import override_settings +from django.urls import URLResolver import zerver.lib.upload +from zerver.lib import cache from zerver.lib.actions import do_set_realm_property -from zerver.lib.upload import S3UploadBackend, LocalUploadBackend from zerver.lib.avatar import avatar_url from zerver.lib.cache import get_cache_backend from zerver.lib.db import Params, ParamsT, Query, TimeTrackingCursor -from zerver.lib import cache -from zerver.tornado import event_queue -from zerver.tornado.handlers import allocate_handler_id -from zerver.worker import queue_processors from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - +from zerver.lib.upload import LocalUploadBackend, S3UploadBackend from zerver.models import ( - get_realm, - get_stream, Client, Message, Realm, Subscription, UserMessage, UserProfile, + get_realm, + get_stream, ) - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict +from zerver.tornado import event_queue +from zerver.tornado.handlers import allocate_handler_id +from zerver.worker import queue_processors +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult if TYPE_CHECKING: # Avoid an import cycle; we only need these for type annotations. from zerver.lib.test_classes import ZulipTestCase, MigrationsTestCase import collections -from unittest import mock import os import re import sys import time -import ujson -from moto import mock_s3 +from unittest import mock import fakeldap import ldap +import ujson +from moto import mock_s3 + class MockLDAP(fakeldap.MockLDAP): class LDAPError(ldap.LDAPError): diff --git a/zerver/lib/test_runner.py b/zerver/lib/test_runner.py index e01590441a..faa2527876 100644 --- a/zerver/lib/test_runner.py +++ b/zerver/lib/test_runner.py @@ -1,38 +1,33 @@ -from functools import partial +import os import random +import shutil import sys - -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, \ - Type, TypeVar, Union +import time +import unittest +from functools import partial +from multiprocessing.sharedctypes import Synchronized +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from unittest import loader, runner from unittest.result import TestResult from django.conf import settings -from django.db import connections, ProgrammingError -from django.urls.resolvers import URLPattern +from django.db import ProgrammingError, connections from django.test import TestCase from django.test import runner as django_runner from django.test.runner import DiscoverRunner from django.test.signals import template_rendered +from django.urls.resolvers import URLPattern +from scripts.lib.zulip_tools import ( + TEMPLATE_DATABASE_DIR, + get_dev_uuid_var_path, + get_or_create_dev_uuid_var_path, +) from zerver.lib import test_helpers from zerver.lib.cache import bounce_key_prefix_for_testing from zerver.lib.rate_limiter import bounce_redis_key_prefix_for_testing from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - write_instrumentation_reports, - append_instrumentation_data, -) - -import os -import time -import unittest -import shutil - -from multiprocessing.sharedctypes import Synchronized - -from scripts.lib.zulip_tools import get_dev_uuid_var_path, TEMPLATE_DATABASE_DIR, \ - get_or_create_dev_uuid_var_path +from zerver.lib.test_helpers import append_instrumentation_data, write_instrumentation_reports # We need to pick an ID for this test-backend invocation, and store it # in this global so it can be used in init_worker; this is used to diff --git a/zerver/lib/tex.py b/zerver/lib/tex.py index 429f2d6d7a..a53f2d4d16 100644 --- a/zerver/lib/tex.py +++ b/zerver/lib/tex.py @@ -1,10 +1,13 @@ import logging import os import subprocess -from django.conf import settings from typing import Optional + +from django.conf import settings + from zerver.lib.storage import static_path + def render_tex(tex: str, is_inline: bool=True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX diff --git a/zerver/lib/thumbnail.py b/zerver/lib/thumbnail.py index 7897a9d5fc..115589a208 100644 --- a/zerver/lib/thumbnail.py +++ b/zerver/lib/thumbnail.py @@ -4,6 +4,7 @@ import os import sys import urllib from urllib.parse import urljoin + from django.conf import settings from django.utils.http import is_safe_url from libthumbor import CryptoURL @@ -11,10 +12,9 @@ from libthumbor import CryptoURL ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) -from zthumbor.loaders.helpers import ( - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, -) from zerver.lib.camo import get_camo_url +from zthumbor.loaders.helpers import THUMBOR_EXTERNAL_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_S3_TYPE + def is_thumbor_enabled() -> bool: return settings.THUMBOR_URL != '' diff --git a/zerver/lib/timeout.py b/zerver/lib/timeout.py index 84418df695..043afe4c9a 100644 --- a/zerver/lib/timeout.py +++ b/zerver/lib/timeout.py @@ -1,11 +1,10 @@ +import ctypes +import sys +import threading +import time from types import TracebackType from typing import Any, Callable, Optional, Tuple, Type, TypeVar -import sys -import time -import ctypes -import threading - # Based on https://code.activestate.com/recipes/483752/ class TimeoutExpired(Exception): diff --git a/zerver/lib/timestamp.py b/zerver/lib/timestamp.py index d344bef6f5..dcbc3dee49 100644 --- a/zerver/lib/timestamp.py +++ b/zerver/lib/timestamp.py @@ -1,5 +1,6 @@ -import datetime import calendar +import datetime + class TimezoneNotUTCException(Exception): pass diff --git a/zerver/lib/timezone.py b/zerver/lib/timezone.py index 949a985682..ac88347c6f 100644 --- a/zerver/lib/timezone.py +++ b/zerver/lib/timezone.py @@ -1,7 +1,8 @@ -from typing import List, Dict, Union, Any +import datetime +from typing import Any, Dict, List, Union import pytz -import datetime + def get_all_timezones() -> List[str]: return sorted(pytz.all_timezones) diff --git a/zerver/lib/topic.py b/zerver/lib/topic.py index b5098f3739..55fe2b504c 100644 --- a/zerver/lib/topic.py +++ b/zerver/lib/topic.py @@ -1,22 +1,11 @@ -from django.db import connection -from django.db.models.query import QuerySet, Q +from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy.sql import ( - column, - literal, - func, -) +from django.db import connection +from django.db.models.query import Q, QuerySet +from sqlalchemy.sql import column, func, literal from zerver.lib.request import REQ -from zerver.models import ( - Message, - Recipient, - Stream, - UserMessage, - UserProfile, -) - -from typing import Any, Dict, List, Optional, Tuple +from zerver.models import Message, Recipient, Stream, UserMessage, UserProfile # Only use these constants for events. ORIG_TOPIC = "orig_subject" diff --git a/zerver/lib/topic_mutes.py b/zerver/lib/topic_mutes.py index 359dbcf0d9..f1a30df323 100644 --- a/zerver/lib/topic_mutes.py +++ b/zerver/lib/topic_mutes.py @@ -1,24 +1,13 @@ -from typing import Any, Callable, Dict, List, Optional, Union import datetime - -from zerver.lib.topic import ( - topic_match_sa, -) -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.models import ( - get_stream, - MutedTopic, - UserProfile, -) -from sqlalchemy.sql import ( - and_, - column, - not_, - or_, - Selectable, -) +from typing import Any, Callable, Dict, List, Optional, Union from django.utils.timezone import now as timezone_now +from sqlalchemy.sql import Selectable, and_, column, not_, or_ + +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.topic import topic_match_sa +from zerver.models import MutedTopic, UserProfile, get_stream + def get_topic_mutes(user_profile: UserProfile) -> List[List[Union[str, float]]]: rows = MutedTopic.objects.filter( diff --git a/zerver/lib/transfer.py b/zerver/lib/transfer.py index cf07727993..5656240b61 100644 --- a/zerver/lib/transfer.py +++ b/zerver/lib/transfer.py @@ -1,14 +1,14 @@ -import os import logging +import os +from mimetypes import guess_type from django.conf import settings from django.db import connection -from mimetypes import guess_type -from zerver.models import UserProfile, Attachment, RealmEmoji from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 from zerver.lib.parallel import run_parallel +from zerver.lib.upload import S3UploadBackend, upload_image_to_s3 +from zerver.models import Attachment, RealmEmoji, UserProfile s3backend = S3UploadBackend() diff --git a/zerver/lib/type_debug.py b/zerver/lib/type_debug.py index 35a07fbc45..fed24fc977 100644 --- a/zerver/lib/type_debug.py +++ b/zerver/lib/type_debug.py @@ -1,7 +1,7 @@ -import sys import functools +import sys +from typing import IO, Any, Callable, Mapping, Sequence, TypeVar -from typing import Any, Callable, IO, Mapping, Sequence, TypeVar def get_mapping_type_str(x: Mapping[Any, Any]) -> str: container_type = type(x).__name__ diff --git a/zerver/lib/types.py b/zerver/lib/types.py index 04eb073207..ab274eeecd 100644 --- a/zerver/lib/types.py +++ b/zerver/lib/types.py @@ -1,6 +1,7 @@ -from typing import TypeVar, Callable, Optional, List, Dict, Union, Tuple, Any -from typing_extensions import TypedDict +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union + from django.http import HttpResponse +from typing_extensions import TypedDict ViewFuncT = TypeVar('ViewFuncT', bound=Callable[..., HttpResponse]) diff --git a/zerver/lib/unminify.py b/zerver/lib/unminify.py index d6b41b720f..3c19ea7a8d 100644 --- a/zerver/lib/unminify.py +++ b/zerver/lib/unminify.py @@ -1,9 +1,9 @@ -import re import os -import sourcemap - +import re from typing import Dict, List +import sourcemap + from zerver.lib.pysa import mark_sanitized diff --git a/zerver/lib/upload.py b/zerver/lib/upload.py index 70f9db9ea6..65aa0278db 100644 --- a/zerver/lib/upload.py +++ b/zerver/lib/upload.py @@ -1,45 +1,45 @@ -from typing import Any, Optional, Tuple - -from datetime import timedelta - -from django.utils.translation import ugettext as _ -from django.conf import settings -from django.core.files import File -from django.core.signing import TimestampSigner, BadSignature -from django.http import HttpRequest -from django.urls import reverse -from jinja2 import Markup as mark_safe +import base64 +import binascii +import io +import logging +import os +import random +import re +import shutil +import sys import unicodedata - -from zerver.lib.avatar_hash import user_avatar_path -from zerver.lib.exceptions import JsonableError, ErrorCode -from zerver.lib.utils import generate_random_token +import urllib +from datetime import timedelta +from mimetypes import guess_extension, guess_type +from typing import Any, Optional, Tuple import boto3 import botocore -from botocore.client import Config from boto3.resources.base import ServiceResource from boto3.session import Session - -from mimetypes import guess_type, guess_extension - -from zerver.models import get_user_profile_by_id -from zerver.models import Attachment -from zerver.models import Realm, RealmEmoji, UserProfile, Message - -import urllib -import base64 -import binascii -import os -import re -from PIL import Image, ImageOps, ExifTags -from PIL.Image import DecompressionBombError +from botocore.client import Config +from django.conf import settings +from django.core.files import File +from django.core.signing import BadSignature, TimestampSigner +from django.http import HttpRequest +from django.urls import reverse +from django.utils.translation import ugettext as _ +from jinja2 import Markup as mark_safe +from PIL import ExifTags, Image, ImageOps from PIL.GifImagePlugin import GifImageFile -import io -import random -import logging -import shutil -import sys +from PIL.Image import DecompressionBombError + +from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.exceptions import ErrorCode, JsonableError +from zerver.lib.utils import generate_random_token +from zerver.models import ( + Attachment, + Message, + Realm, + RealmEmoji, + UserProfile, + get_user_profile_by_id, +) DEFAULT_AVATAR_SIZE = 100 MEDIUM_AVATAR_SIZE = 500 diff --git a/zerver/lib/url_encoding.py b/zerver/lib/url_encoding.py index 3bd12dd276..6c8058b90f 100644 --- a/zerver/lib/url_encoding.py +++ b/zerver/lib/url_encoding.py @@ -5,6 +5,7 @@ from zerver.lib.pysa import mark_sanitized from zerver.lib.topic import get_topic_from_message_info from zerver.models import Realm, Stream, UserProfile + def hash_util_encode(string: str) -> str: # Do the same encoding operation as hash_util.encodeHashComponent on the # frontend. diff --git a/zerver/lib/url_preview/oembed.py b/zerver/lib/url_preview/oembed.py index 16831f50a2..b9451806b1 100644 --- a/zerver/lib/url_preview/oembed.py +++ b/zerver/lib/url_preview/oembed.py @@ -1,6 +1,8 @@ -from typing import Optional, Dict, Any -from pyoembed import oEmbed, PyOembedException import json +from typing import Any, Dict, Optional + +from pyoembed import PyOembedException, oEmbed + def get_oembed_data(url: str, maxwidth: Optional[int]=640, diff --git a/zerver/lib/url_preview/parsers/__init__.py b/zerver/lib/url_preview/parsers/__init__.py index 74168a2c3b..b812d115a0 100644 --- a/zerver/lib/url_preview/parsers/__init__.py +++ b/zerver/lib/url_preview/parsers/__init__.py @@ -1,4 +1,4 @@ -from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser from zerver.lib.url_preview.parsers.generic import GenericParser +from zerver.lib.url_preview.parsers.open_graph import OpenGraphParser __all__ = ['OpenGraphParser', 'GenericParser'] diff --git a/zerver/lib/url_preview/parsers/base.py b/zerver/lib/url_preview/parsers/base.py index 25ed9e8ff6..b173221302 100644 --- a/zerver/lib/url_preview/parsers/base.py +++ b/zerver/lib/url_preview/parsers/base.py @@ -1,5 +1,6 @@ from typing import Any + class BaseParser: def __init__(self, html_source: str) -> None: # We import BeautifulSoup here, because it's not used by most diff --git a/zerver/lib/url_preview/parsers/generic.py b/zerver/lib/url_preview/parsers/generic.py index a57db1664e..d938114f36 100644 --- a/zerver/lib/url_preview/parsers/generic.py +++ b/zerver/lib/url_preview/parsers/generic.py @@ -1,4 +1,5 @@ from typing import Dict, Optional + from zerver.lib.url_preview.parsers.base import BaseParser diff --git a/zerver/lib/url_preview/parsers/open_graph.py b/zerver/lib/url_preview/parsers/open_graph.py index 75eee049e2..8c31546e01 100644 --- a/zerver/lib/url_preview/parsers/open_graph.py +++ b/zerver/lib/url_preview/parsers/open_graph.py @@ -1,4 +1,5 @@ from typing import Dict + from .base import BaseParser diff --git a/zerver/lib/url_preview/preview.py b/zerver/lib/url_preview/preview.py index f69d574043..f764744100 100644 --- a/zerver/lib/url_preview/preview.py +++ b/zerver/lib/url_preview/preview.py @@ -1,17 +1,17 @@ import re -import requests +from typing import Any, Callable, Dict, Optional +from typing.re import Match +import magic +import requests from django.conf import settings from django.utils.encoding import smart_text -import magic -from typing import Any, Optional, Dict, Callable -from typing.re import Match from version import ZULIP_VERSION from zerver.lib.cache import cache_with_key, get_cache_with_key, preview_url_cache_key from zerver.lib.pysa import mark_sanitized from zerver.lib.url_preview.oembed import get_oembed_data -from zerver.lib.url_preview.parsers import OpenGraphParser, GenericParser +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser # FIXME: Should we use a database cache or a memcached in production? What if # opengraph data is changed for a site? diff --git a/zerver/lib/user_groups.py b/zerver/lib/user_groups.py index 9aec110b3c..5ce7209dc4 100644 --- a/zerver/lib/user_groups.py +++ b/zerver/lib/user_groups.py @@ -1,8 +1,11 @@ +from typing import Any, Dict, List + from django.db import transaction from django.utils.translation import ugettext as _ + from zerver.lib.exceptions import JsonableError -from zerver.models import UserProfile, Realm, UserGroupMembership, UserGroup -from typing import Dict, List, Any +from zerver.models import Realm, UserGroup, UserGroupMembership, UserProfile + def access_user_group_by_id(user_group_id: int, user_profile: UserProfile) -> UserGroup: try: diff --git a/zerver/lib/user_status.py b/zerver/lib/user_status.py index fa2cca500b..da4aa90f39 100644 --- a/zerver/lib/user_status.py +++ b/zerver/lib/user_status.py @@ -1,11 +1,10 @@ +from typing import Any, Dict, Optional + from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.models import ( - UserStatus, -) +from zerver.models import UserStatus -from typing import Any, Dict, Optional def get_user_info_dict(realm_id: int) -> Dict[int, Dict[str, Any]]: rows = UserStatus.objects.filter( diff --git a/zerver/lib/users.py b/zerver/lib/users.py index 61fe52318e..d92024540c 100644 --- a/zerver/lib/users.py +++ b/zerver/lib/users.py @@ -1,25 +1,34 @@ -from typing import Any, Dict, List, Optional, Tuple, Union, cast - import re import unicodedata from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple, Union, cast from django.conf import settings from django.db.models.query import QuerySet from django.forms.models import model_to_dict from django.utils.translation import ugettext as _ - -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_cache_key_id, \ - user_profile_by_id_cache_key, realm_user_dict_fields -from zerver.lib.request import JsonableError -from zerver.lib.avatar import avatar_url, get_avatar_field -from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.models import UserProfile, Service, Realm, \ - get_user_profile_by_id_in_realm, CustomProfileFieldValue, \ - get_realm_user_dicts, CustomProfileField - from zulip_bots.custom_exceptions import ConfigValidationError +from zerver.lib.avatar import avatar_url, get_avatar_field +from zerver.lib.cache import ( + generic_bulk_cached_fetch, + realm_user_dict_fields, + user_profile_by_id_cache_key, + user_profile_cache_key_id, +) +from zerver.lib.exceptions import OrganizationAdministratorRequired +from zerver.lib.request import JsonableError +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + Realm, + Service, + UserProfile, + get_realm_user_dicts, + get_user_profile_by_id_in_realm, +) + + def check_full_name(full_name_raw: str) -> str: full_name = full_name_raw.strip() if len(full_name) > UserProfile.MAX_NAME_LENGTH: diff --git a/zerver/lib/utils.py b/zerver/lib/utils.py index 4b1d35ef8e..59926fc8a8 100644 --- a/zerver/lib/utils.py +++ b/zerver/lib/utils.py @@ -1,4 +1,3 @@ -from typing import Any, Callable, List, Optional, Sequence, TypeVar, Iterable, Set, Tuple import base64 import hashlib import heapq @@ -6,8 +5,9 @@ import itertools import os import re import string -from time import sleep from itertools import zip_longest +from time import sleep +from typing import Any, Callable, Iterable, List, Optional, Sequence, Set, Tuple, TypeVar from django.conf import settings diff --git a/zerver/lib/validator.py b/zerver/lib/validator.py index 00920b0706..59f4e24b61 100644 --- a/zerver/lib/validator.py +++ b/zerver/lib/validator.py @@ -26,17 +26,17 @@ To extend this concept, it's simply a matter of writing your own validator for any particular type of object. ''' import re +from datetime import datetime +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union, cast + import ujson -from django.utils.translation import ugettext as _ from django.conf import settings from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator -from typing import Any, Dict, Iterable, Optional, Tuple, cast, List, Callable, TypeVar, \ - Set, Union +from django.core.validators import URLValidator, validate_email +from django.utils.translation import ugettext as _ -from datetime import datetime from zerver.lib.request import JsonableError -from zerver.lib.types import Validator, ProfileFieldData +from zerver.lib.types import ProfileFieldData, Validator FuncT = Callable[..., Any] TypeStructure = TypeVar("TypeStructure") diff --git a/zerver/lib/webhooks/common.py b/zerver/lib/webhooks/common.py index 442d5e406f..3edd2ca0d2 100644 --- a/zerver/lib/webhooks/common.py +++ b/zerver/lib/webhooks/common.py @@ -1,19 +1,25 @@ import importlib +from typing import Any, Callable, Dict, Optional, Union from urllib.parse import unquote from django.http import HttpRequest from django.utils.translation import ugettext as _ -from typing import Optional, Dict, Union, Any, Callable -from zerver.lib.actions import check_send_stream_message, \ - check_send_private_message, send_rate_limited_pm_notification_to_bot_owner -from zerver.lib.exceptions import StreamDoesNotExistError, JsonableError, \ - ErrorCode, UnexpectedWebhookEventType +from zerver.lib.actions import ( + check_send_private_message, + check_send_stream_message, + send_rate_limited_pm_notification_to_bot_owner, +) +from zerver.lib.exceptions import ( + ErrorCode, + JsonableError, + StreamDoesNotExistError, + UnexpectedWebhookEventType, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.send_email import FromAddress from zerver.models import UserProfile - MISSING_EVENT_HEADER_MESSAGE = """ Hi there! Your bot {bot_name} just sent an HTTP request to {request_path} that is missing the HTTP {header_name} header. Because this header is how diff --git a/zerver/lib/webhooks/git.py b/zerver/lib/webhooks/git.py index b57f33b843..84b548e888 100644 --- a/zerver/lib/webhooks/git.py +++ b/zerver/lib/webhooks/git.py @@ -1,6 +1,7 @@ import string -from typing import Optional, Any, Dict, List, Tuple from collections import defaultdict +from typing import Any, Dict, List, Optional, Tuple + TOPIC_WITH_BRANCH_TEMPLATE = '{repo} / {branch}' TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE = '{repo} / {type} #{id} {title}' TOPIC_WITH_RELEASE_TEMPLATE = '{repo} / {tag} {title}' diff --git a/zerver/lib/widget.py b/zerver/lib/widget.py index 319242c43b..fa3ad9cc3d 100644 --- a/zerver/lib/widget.py +++ b/zerver/lib/widget.py @@ -1,7 +1,6 @@ -from typing import MutableMapping, Any, Optional, Tuple - -import re import json +import re +from typing import Any, MutableMapping, Optional, Tuple from zerver.models import SubMessage diff --git a/zerver/lib/zcommand.py b/zerver/lib/zcommand.py index d32253368f..f6909b347d 100644 --- a/zerver/lib/zcommand.py +++ b/zerver/lib/zcommand.py @@ -1,9 +1,11 @@ from typing import Any, Dict + from django.utils.translation import ugettext as _ -from zerver.models import UserProfile from zerver.lib.actions import do_set_user_display_setting from zerver.lib.exceptions import JsonableError +from zerver.models import UserProfile + def process_zcommands(content: str, user_profile: UserProfile) -> Dict[str, Any]: def change_mode_setting(command: str, switch_command: str, diff --git a/zerver/lib/zephyr.py b/zerver/lib/zephyr.py index c39b2f52a7..116ae1beae 100644 --- a/zerver/lib/zephyr.py +++ b/zerver/lib/zephyr.py @@ -1,7 +1,9 @@ import re import traceback + import DNS + def compute_mit_user_fullname(email: str) -> str: try: # Input is either e.g. username@mit.edu or user|CROSSREALM.INVALID@mit.edu diff --git a/zerver/logging_handlers.py b/zerver/logging_handlers.py index 5b6539a0f0..19cb2bb18b 100644 --- a/zerver/logging_handlers.py +++ b/zerver/logging_handlers.py @@ -1,8 +1,7 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - import logging -import platform import os +import platform import subprocess import traceback from typing import Any, Dict, Optional @@ -11,9 +10,10 @@ from django.conf import settings from django.http import HttpRequest from django.views.debug import get_exception_reporter_filter +from version import ZULIP_VERSION from zerver.lib.logging_util import find_log_caller_module from zerver.lib.queue import queue_json_publish -from version import ZULIP_VERSION + def try_git_describe() -> Optional[str]: try: # nocoverage diff --git a/zerver/management/commands/check_redis.py b/zerver/management/commands/check_redis.py index d523de3f86..bc4cd91d1a 100644 --- a/zerver/management/commands/check_redis.py +++ b/zerver/management/commands/check_redis.py @@ -3,8 +3,7 @@ import time from typing import Any, Callable, Optional from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.rate_limiter import RateLimitedUser, client from zerver.models import get_user_profile_by_id diff --git a/zerver/management/commands/convert_gitter_data.py b/zerver/management/commands/convert_gitter_data.py index 62e88568c5..e18bd893b9 100644 --- a/zerver/management/commands/convert_gitter_data.py +++ b/zerver/management/commands/convert_gitter_data.py @@ -3,8 +3,7 @@ import os import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.gitter import do_convert_data diff --git a/zerver/management/commands/convert_hipchat_data.py b/zerver/management/commands/convert_hipchat_data.py index 9061c44766..e3f4e77cfe 100644 --- a/zerver/management/commands/convert_hipchat_data.py +++ b/zerver/management/commands/convert_hipchat_data.py @@ -21,8 +21,7 @@ spec: exporting-from-hipchat-server-or-data-center-for-data-portability-950821555.html ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.hipchat import do_convert_data diff --git a/zerver/management/commands/convert_mattermost_data.py b/zerver/management/commands/convert_mattermost_data.py index d0e1df3bdd..292a8b9c3d 100644 --- a/zerver/management/commands/convert_mattermost_data.py +++ b/zerver/management/commands/convert_mattermost_data.py @@ -14,8 +14,7 @@ Test out the realm: go to browser and use your dev url ''' -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.mattermost import do_convert_data diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 7a75f0bbed..5168146d14 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -3,8 +3,7 @@ import os import tempfile from typing import Any -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.data_import.slack import do_convert_data diff --git a/zerver/management/commands/generate_invite_links.py b/zerver/management/commands/generate_invite_links.py index d6fdbe05fa..cfb832469d 100644 --- a/zerver/management/commands/generate_invite_links.py +++ b/zerver/management/commands/generate_invite_links.py @@ -4,9 +4,9 @@ from typing import Any from django.core.management.base import CommandError from confirmation.models import Confirmation, create_confirmation_link +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.management import ZulipBaseCommand from zerver.models import DomainNotAllowedForRealmError, PreregistrationUser -from zerver.lib.email_validation import email_allowed_for_realm class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/import.py b/zerver/management/commands/import.py index 1f3406ecd8..92282013c8 100644 --- a/zerver/management/commands/import.py +++ b/zerver/management/commands/import.py @@ -5,8 +5,7 @@ from typing import Any from django.conf import settings from django.core.management import call_command -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.forms import check_subdomain_available from zerver.lib.import_realm import do_import_realm, do_import_system_bots diff --git a/zerver/management/commands/knight.py b/zerver/management/commands/knight.py index 51194ae72b..61532c355a 100644 --- a/zerver/management/commands/knight.py +++ b/zerver/management/commands/knight.py @@ -3,7 +3,7 @@ from typing import Any from django.core.management.base import CommandError -from zerver.lib.actions import do_change_user_role, do_change_is_api_super_user +from zerver.lib.actions import do_change_is_api_super_user, do_change_user_role from zerver.lib.management import ZulipBaseCommand from zerver.models import UserProfile diff --git a/zerver/management/commands/logout_all_users.py b/zerver/management/commands/logout_all_users.py index 6108643a58..b41b1c867c 100644 --- a/zerver/management/commands/logout_all_users.py +++ b/zerver/management/commands/logout_all_users.py @@ -2,8 +2,11 @@ from argparse import ArgumentParser from typing import Any from zerver.lib.management import ZulipBaseCommand -from zerver.lib.sessions import delete_all_deactivated_user_sessions, \ - delete_all_user_sessions, delete_realm_user_sessions +from zerver.lib.sessions import ( + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, +) class Command(ZulipBaseCommand): diff --git a/zerver/management/commands/makemessages.py b/zerver/management/commands/makemessages.py index 80f6220fd7..69ac464471 100644 --- a/zerver/management/commands/makemessages.py +++ b/zerver/management/commands/makemessages.py @@ -29,8 +29,8 @@ https://stackoverflow.com/questions/2090717 """ import glob -import json import itertools +import json import os import re from argparse import ArgumentParser diff --git a/zerver/management/commands/merge_streams.py b/zerver/management/commands/merge_streams.py index 517446da72..5d56f9e5a1 100644 --- a/zerver/management/commands/merge_streams.py +++ b/zerver/management/commands/merge_streams.py @@ -1,8 +1,11 @@ from argparse import ArgumentParser from typing import Any, List -from zerver.lib.actions import bulk_add_subscriptions, \ - bulk_remove_subscriptions, do_deactivate_stream +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_deactivate_stream, +) from zerver.lib.cache import cache_delete_many, to_dict_cache_key_id from zerver.lib.management import ZulipBaseCommand from zerver.models import Message, Subscription, get_stream diff --git a/zerver/management/commands/restore_messages.py b/zerver/management/commands/restore_messages.py index 9570658d01..fa0b2a8b20 100644 --- a/zerver/management/commands/restore_messages.py +++ b/zerver/management/commands/restore_messages.py @@ -3,8 +3,11 @@ from typing import Any from django.core.management.base import CommandParser from zerver.lib.management import ZulipBaseCommand -from zerver.lib.retention import restore_all_data_from_archive, \ - restore_data_from_archive, restore_data_from_archive_by_realm +from zerver.lib.retention import ( + restore_all_data_from_archive, + restore_data_from_archive, + restore_data_from_archive_by_realm, +) from zerver.models import ArchiveTransaction diff --git a/zerver/management/commands/runtornado.py b/zerver/management/commands/runtornado.py index 02e27760c0..de3adfe1ad 100644 --- a/zerver/management/commands/runtornado.py +++ b/zerver/management/commands/runtornado.py @@ -3,8 +3,7 @@ import sys from typing import Any, Callable from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from tornado import ioloop from tornado.log import app_log @@ -19,11 +18,14 @@ settings.RUNNING_INSIDE_TORNADO = True instrument_tornado_ioloop() from zerver.lib.debug import interactive_debug_listen -from zerver.tornado.application import create_tornado_application, \ - setup_tornado_rabbitmq +from zerver.tornado.application import create_tornado_application, setup_tornado_rabbitmq from zerver.tornado.autoreload import start as zulip_autoreload_start -from zerver.tornado.event_queue import add_client_gc_hook, \ - missedmessage_hook, get_wrapped_process_notification, setup_event_queue +from zerver.tornado.event_queue import ( + add_client_gc_hook, + get_wrapped_process_notification, + missedmessage_hook, + setup_event_queue, +) from zerver.tornado.sharding import notify_tornado_queue_name if settings.USING_RABBITMQ: diff --git a/zerver/management/commands/show_admins.py b/zerver/management/commands/show_admins.py index 7962adaee5..f94479ec6a 100644 --- a/zerver/management/commands/show_admins.py +++ b/zerver/management/commands/show_admins.py @@ -2,8 +2,10 @@ from argparse import ArgumentParser from typing import Any from django.core.management.base import CommandError + from zerver.lib.management import ZulipBaseCommand + class Command(ZulipBaseCommand): help = """Show the owners and administrators in an organization.""" diff --git a/zerver/management/commands/soft_deactivate_users.py b/zerver/management/commands/soft_deactivate_users.py index f379cd9edd..26481ab1ac 100644 --- a/zerver/management/commands/soft_deactivate_users.py +++ b/zerver/management/commands/soft_deactivate_users.py @@ -6,8 +6,12 @@ from django.conf import settings from django.core.management.base import CommandError from zerver.lib.management import ZulipBaseCommand -from zerver.lib.soft_deactivation import do_auto_soft_deactivate_users, \ - do_soft_activate_users, do_soft_deactivate_users, logger +from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_soft_activate_users, + do_soft_deactivate_users, + logger, +) from zerver.models import Realm, UserProfile diff --git a/zerver/management/commands/transfer_uploads_to_s3.py b/zerver/management/commands/transfer_uploads_to_s3.py index d2c70f9f38..77ade9cf48 100644 --- a/zerver/management/commands/transfer_uploads_to_s3.py +++ b/zerver/management/commands/transfer_uploads_to_s3.py @@ -1,8 +1,7 @@ from typing import Any from django.conf import settings -from django.core.management.base import BaseCommand, CommandError, \ - CommandParser +from django.core.management.base import BaseCommand, CommandError, CommandParser from zerver.lib.transfer import transfer_uploads_to_s3 diff --git a/zerver/middleware.py b/zerver/middleware.py index 3ceeb11576..95795b64b3 100644 --- a/zerver/middleware.py +++ b/zerver/middleware.py @@ -2,8 +2,7 @@ import cProfile import logging import time import traceback -from typing import Any, AnyStr, Dict, \ - Iterable, List, MutableMapping, Optional +from typing import Any, AnyStr, Dict, Iterable, List, MutableMapping, Optional from django.conf import settings from django.core.exceptions import DisallowedHost @@ -17,15 +16,15 @@ from django.views.csrf import csrf_failure as html_csrf_failure from zerver.lib.bugdown import get_bugdown_requests, get_bugdown_time from zerver.lib.cache import get_remote_cache_requests, get_remote_cache_time -from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.db import reset_queries +from zerver.lib.debug import maybe_tracemalloc_listen from zerver.lib.exceptions import ErrorCode, JsonableError, RateLimited from zerver.lib.html_to_text import get_content_description from zerver.lib.rate_limiter import RateLimitResult from zerver.lib.response import json_error, json_response_from_error from zerver.lib.subdomains import get_subdomain -from zerver.lib.utils import statsd from zerver.lib.types import ViewFuncT +from zerver.lib.utils import statsd from zerver.models import Realm, flush_per_request_caches, get_realm logger = logging.getLogger('zulip.requests') diff --git a/zerver/migrations/0030_realm_org_type.py b/zerver/migrations/0030_realm_org_type.py index 6f32a07e1e..b086b48584 100644 --- a/zerver/migrations/0030_realm_org_type.py +++ b/zerver/migrations/0030_realm_org_type.py @@ -1,6 +1,5 @@ from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0032_verify_all_medium_avatar_images.py b/zerver/migrations/0032_verify_all_medium_avatar_images.py index f477ca52f7..ceb86eeb5a 100644 --- a/zerver/migrations/0032_verify_all_medium_avatar_images.py +++ b/zerver/migrations/0032_verify_all_medium_avatar_images.py @@ -1,11 +1,11 @@ import hashlib from typing import Text +from unittest.mock import patch from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from unittest.mock import patch from zerver.lib.upload import upload_backend from zerver.lib.utils import make_safe_digest diff --git a/zerver/migrations/0038_realm_change_to_community_defaults.py b/zerver/migrations/0038_realm_change_to_community_defaults.py index ad8c885a03..50f65f00e5 100644 --- a/zerver/migrations/0038_realm_change_to_community_defaults.py +++ b/zerver/migrations/0038_realm_change_to_community_defaults.py @@ -1,6 +1,5 @@ from django.db import migrations, models - COMMUNITY = 2 class Migration(migrations.Migration): diff --git a/zerver/migrations/0060_move_avatars_to_be_uid_based.py b/zerver/migrations/0060_move_avatars_to_be_uid_based.py index 6200d286a7..8d834eb38a 100644 --- a/zerver/migrations/0060_move_avatars_to_be_uid_based.py +++ b/zerver/migrations/0060_move_avatars_to_be_uid_based.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-02-27 17:03 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0064_sync_uploads_filesize_with_db.py b/zerver/migrations/0064_sync_uploads_filesize_with_db.py index 674e98f237..337e1c6a81 100644 --- a/zerver/migrations/0064_sync_uploads_filesize_with_db.py +++ b/zerver/migrations/0064_sync_uploads_filesize_with_db.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-18 12:38 from django.db import migrations + class Migration(migrations.Migration): dependencies = [ diff --git a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py index 62f513ec91..3d475d118c 100644 --- a/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py +++ b/zerver/migrations/0077_add_file_name_field_to_realm_emoji.py @@ -1,6 +1,7 @@ # Generated by Django 1.10.5 on 2017-03-09 05:23 from django.db import migrations, models + class Migration(migrations.Migration): dependencies = [ ('zerver', '0076_userprofile_emojiset'), diff --git a/zerver/migrations/0086_realm_alter_default_org_type.py b/zerver/migrations/0086_realm_alter_default_org_type.py index 1dca4623d8..6b5096d180 100644 --- a/zerver/migrations/0086_realm_alter_default_org_type.py +++ b/zerver/migrations/0086_realm_alter_default_org_type.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.2 on 2017-06-26 21:56 from django.db import migrations, models - CORPORATE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0143_realm_bot_creation_policy.py b/zerver/migrations/0143_realm_bot_creation_policy.py index 93414f9522..7b8e7f0faa 100644 --- a/zerver/migrations/0143_realm_bot_creation_policy.py +++ b/zerver/migrations/0143_realm_bot_creation_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.6 on 2018-03-09 18:00 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - BOT_CREATION_EVERYONE = 1 def set_initial_value_for_bot_creation_policy(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model("zerver", "Realm") diff --git a/zerver/migrations/0185_realm_plan_type.py b/zerver/migrations/0185_realm_plan_type.py index 3e9935ebcb..c7349655df 100644 --- a/zerver/migrations/0185_realm_plan_type.py +++ b/zerver/migrations/0185_realm_plan_type.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.14 on 2018-08-10 21:36 - from django.db import migrations, models - SELF_HOSTED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0214_realm_invite_to_stream_policy.py b/zerver/migrations/0214_realm_invite_to_stream_policy.py index 75e2fbbf56..f9d3d85f2a 100644 --- a/zerver/migrations/0214_realm_invite_to_stream_policy.py +++ b/zerver/migrations/0214_realm_invite_to_stream_policy.py @@ -1,10 +1,8 @@ # Generated by Django 1.11.20 on 2019-04-29 05:29 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps - INVITE_TO_STREAM_POLICY_MEMBERS = 1 def handle_waiting_period(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: Realm = apps.get_model('zerver', 'Realm') diff --git a/zerver/migrations/0228_userprofile_demote_inactive_streams.py b/zerver/migrations/0228_userprofile_demote_inactive_streams.py index 30ac8d84fa..bdb779d0b0 100644 --- a/zerver/migrations/0228_userprofile_demote_inactive_streams.py +++ b/zerver/migrations/0228_userprofile_demote_inactive_streams.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-03-08 19:50 - from django.db import migrations, models - DEMOTE_STREAMS_AUTOMATIC = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py index 8ff03ae5b1..c62476ff87 100644 --- a/zerver/migrations/0235_userprofile_desktop_icon_count_display.py +++ b/zerver/migrations/0235_userprofile_desktop_icon_count_display.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.20 on 2019-06-29 18:22 - from django.db import migrations, models - DESKTOP_ICON_COUNT_DISPLAY_MESSAGES = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0248_userprofile_role_start.py b/zerver/migrations/0248_userprofile_role_start.py index 0306767867..9408ef52dd 100644 --- a/zerver/migrations/0248_userprofile_role_start.py +++ b/zerver/migrations/0248_userprofile_role_start.py @@ -1,9 +1,9 @@ # Generated by Django 1.11.24 on 2019-10-03 22:27 - from django.db import migrations, models from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps + def update_role(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') # The values at the time of this migration diff --git a/zerver/migrations/0252_realm_user_group_edit_policy.py b/zerver/migrations/0252_realm_user_group_edit_policy.py index 25a1e61ee1..193fc7b1dd 100644 --- a/zerver/migrations/0252_realm_user_group_edit_policy.py +++ b/zerver/migrations/0252_realm_user_group_edit_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.24 on 2019-10-16 22:48 - from django.db import migrations, models - USER_GROUP_EDIT_POLICY_MEMBERS = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0261_realm_private_message_policy.py b/zerver/migrations/0261_realm_private_message_policy.py index 33a056a81e..8688872bcc 100644 --- a/zerver/migrations/0261_realm_private_message_policy.py +++ b/zerver/migrations/0261_realm_private_message_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-08 00:32 - from django.db import migrations, models - PRIVATE_MESSAGE_POLICY_UNLIMITED = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0262_mutedtopic_date_muted.py b/zerver/migrations/0262_mutedtopic_date_muted.py index 22ee32a792..f42dd83e2b 100644 --- a/zerver/migrations/0262_mutedtopic_date_muted.py +++ b/zerver/migrations/0262_mutedtopic_date_muted.py @@ -1,6 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-17 15:26 - import datetime + from django.db import migrations, models diff --git a/zerver/migrations/0263_stream_stream_post_policy.py b/zerver/migrations/0263_stream_stream_post_policy.py index 6cf2945ef9..e619f5c859 100644 --- a/zerver/migrations/0263_stream_stream_post_policy.py +++ b/zerver/migrations/0263_stream_stream_post_policy.py @@ -1,8 +1,6 @@ # Generated by Django 1.11.26 on 2020-01-27 22:03 - from django.db import migrations, models - STREAM_POST_POLICY_EVERYONE = 1 class Migration(migrations.Migration): diff --git a/zerver/migrations/0266_userpresence_realm.py b/zerver/migrations/0266_userpresence_realm.py index 1937e5339c..4d0d7cae9e 100644 --- a/zerver/migrations/0266_userpresence_realm.py +++ b/zerver/migrations/0266_userpresence_realm.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:19 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py index 2eeb6fe187..9c28d38376 100644 --- a/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py +++ b/zerver/migrations/0268_add_userpresence_realm_timestamp_index.py @@ -1,7 +1,6 @@ # Generated by Django 1.11.28 on 2020-02-08 20:34 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0270_huddle_recipient.py b/zerver/migrations/0270_huddle_recipient.py index 9eac7771d7..3273c63d74 100644 --- a/zerver/migrations/0270_huddle_recipient.py +++ b/zerver/migrations/0270_huddle_recipient.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.10 on 2020-03-15 17:25 - -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/zerver/migrations/0273_migrate_old_bot_messages.py b/zerver/migrations/0273_migrate_old_bot_messages.py index 7cd7a0e864..9494b09318 100644 --- a/zerver/migrations/0273_migrate_old_bot_messages.py +++ b/zerver/migrations/0273_migrate_old_bot_messages.py @@ -1,9 +1,10 @@ +from typing import Any + from django.conf import settings from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -from typing import Any def fix_messages(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: """Conceptually, this migration cleans up the old NEW_USER_BOT and FEEDBACK_BOT diff --git a/zerver/migrations/0276_alertword.py b/zerver/migrations/0276_alertword.py index 75d3eafb3c..4de29acc54 100644 --- a/zerver/migrations/0276_alertword.py +++ b/zerver/migrations/0276_alertword.py @@ -1,6 +1,6 @@ +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/zerver/migrations/0277_migrate_alert_word.py b/zerver/migrations/0277_migrate_alert_word.py index c4f755bde7..cca8edc8ef 100644 --- a/zerver/migrations/0277_migrate_alert_word.py +++ b/zerver/migrations/0277_migrate_alert_word.py @@ -1,8 +1,10 @@ +from typing import Dict, List + +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps -import ujson -from typing import Dict, List + def move_to_seperate_table(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py index 8e7463ffda..bbd56d5e0e 100644 --- a/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py +++ b/zerver/migrations/0284_convert_realm_admins_to_realm_owners.py @@ -1,12 +1,13 @@ # Generated by Django 2.2.12 on 2020-05-16 18:34 from typing import Any, Dict +import ujson from django.db import migrations from django.db.backends.postgresql.schema import DatabaseSchemaEditor from django.db.migrations.state import StateApps from django.db.models import Count from django.utils.timezone import now as timezone_now -import ujson + def set_realm_admins_as_realm_owners(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None: UserProfile = apps.get_model('zerver', 'UserProfile') diff --git a/zerver/models.py b/zerver/models.py index d10c1d3ac4..6e3fd7969c 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -1,52 +1,92 @@ -from typing import Any, DefaultDict, Dict, List, Set, Tuple, TypeVar, \ - Union, Optional, Sequence, AbstractSet, Callable, Iterable - -from django.db import models -from django.db.models.query import QuerySet -from django.db.models import Manager, Q, Sum, CASCADE -from django.conf import settings -from django.contrib.auth.models import AbstractBaseUser, UserManager, \ - PermissionsMixin -import django.contrib.auth -from django.core.exceptions import ValidationError -from django.core.validators import URLValidator, MinLengthValidator, \ - RegexValidator, validate_email -from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \ - user_profile_by_api_key_cache_key, active_non_guest_user_ids_cache_key, \ - user_profile_by_id_cache_key, user_profile_by_email_cache_key, \ - user_profile_cache_key, generic_bulk_cached_fetch, cache_set, flush_stream, \ - cache_delete, active_user_ids_cache_key, \ - get_stream_cache_key, realm_user_dicts_cache_key, \ - bot_dicts_in_realm_cache_key, realm_user_dict_fields, \ - bot_dict_fields, flush_message, flush_submessage, bot_profile_cache_key, \ - flush_used_upload_space_cache, get_realm_used_upload_space_cache_key, \ - realm_alert_words_cache_key, realm_alert_words_automaton_cache_key -from zerver.lib.utils import make_safe_digest, generate_random_token -from django.db import transaction -from django.utils.timezone import now as timezone_now -from zerver.lib.timestamp import datetime_to_timestamp -from django.db.models.signals import post_save, post_delete -from django.utils.translation import ugettext_lazy as _ -from zerver.lib import cache -from zerver.lib.pysa import mark_sanitized -from zerver.lib.validator import check_int, \ - check_short_string, check_long_string, validate_choice_field, check_date, \ - check_url, check_list -from zerver.lib.types import Validator, ExtendedValidator, \ - ProfileDataElement, ProfileData, RealmUserValidator, \ - ExtendedFieldElement, UserFieldElement, FieldElement, \ - DisplayRecipientT -from zerver.lib.exceptions import JsonableError -from django.contrib.postgres.fields import JSONField - -from bitfield import BitField -from bitfield.types import BitHandler -from collections import defaultdict -from datetime import timedelta +import datetime import re import sre_constants import time -import datetime +from collections import defaultdict +from datetime import timedelta +from typing import ( + AbstractSet, + Any, + Callable, + DefaultDict, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + TypeVar, + Union, +) + +import django.contrib.auth +from bitfield import BitField +from bitfield.types import BitHandler +from django.conf import settings +from django.contrib.auth.models import AbstractBaseUser, PermissionsMixin, UserManager +from django.contrib.postgres.fields import JSONField +from django.core.exceptions import ValidationError +from django.core.validators import MinLengthValidator, RegexValidator, URLValidator, validate_email +from django.db import models, transaction +from django.db.models import CASCADE, Manager, Q, Sum +from django.db.models.query import QuerySet +from django.db.models.signals import post_delete, post_save +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext_lazy as _ + +from zerver.lib import cache +from zerver.lib.cache import ( + active_non_guest_user_ids_cache_key, + active_user_ids_cache_key, + bot_dict_fields, + bot_dicts_in_realm_cache_key, + bot_profile_cache_key, + cache_delete, + cache_set, + cache_with_key, + flush_message, + flush_realm, + flush_stream, + flush_submessage, + flush_used_upload_space_cache, + flush_user_profile, + generic_bulk_cached_fetch, + get_realm_used_upload_space_cache_key, + get_stream_cache_key, + realm_alert_words_automaton_cache_key, + realm_alert_words_cache_key, + realm_user_dict_fields, + realm_user_dicts_cache_key, + user_profile_by_api_key_cache_key, + user_profile_by_email_cache_key, + user_profile_by_id_cache_key, + user_profile_cache_key, +) +from zerver.lib.exceptions import JsonableError +from zerver.lib.pysa import mark_sanitized +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.types import ( + DisplayRecipientT, + ExtendedFieldElement, + ExtendedValidator, + FieldElement, + ProfileData, + ProfileDataElement, + RealmUserValidator, + UserFieldElement, + Validator, +) +from zerver.lib.utils import generate_random_token, make_safe_digest +from zerver.lib.validator import ( + check_date, + check_int, + check_list, + check_long_string, + check_short_string, + check_url, + validate_choice_field, +) MAX_TOPIC_NAME_LENGTH = 60 MAX_MESSAGE_LENGTH = 10000 diff --git a/zerver/openapi/curl_param_value_generators.py b/zerver/openapi/curl_param_value_generators.py index 59cb4d68e0..b6ebadd67a 100644 --- a/zerver/openapi/curl_param_value_generators.py +++ b/zerver/openapi/curl_param_value_generators.py @@ -1,13 +1,12 @@ -from typing import Dict, Any, Callable, Set, List, Optional, Tuple - from functools import wraps +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from django.utils.timezone import now as timezone_now -from zerver.models import Client, Message, UserPresence, UserGroup, get_realm -from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.actions import do_add_reaction, do_add_realm_filter, update_user_presence from zerver.lib.events import do_events_register -from zerver.lib.actions import update_user_presence, do_add_realm_filter, do_add_reaction +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Client, Message, UserGroup, UserPresence, get_realm GENERATOR_FUNCTIONS: Dict[str, Callable[..., Dict[Any, Any]]] = dict() REGISTERED_GENERATOR_FUNCTIONS: Set[str] = set() diff --git a/zerver/openapi/javascript_examples.py b/zerver/openapi/javascript_examples.py index cbb41a1457..47318cda2b 100644 --- a/zerver/openapi/javascript_examples.py +++ b/zerver/openapi/javascript_examples.py @@ -1,10 +1,12 @@ -import os import json +import os import subprocess from zulip import Client + from zerver.openapi.openapi import validate_against_openapi_schema + def test_js_bindings(client: Client) -> None: os.environ['ZULIP_USERNAME'] = client.email os.environ['ZULIP_API_KEY'] = client.api_key diff --git a/zerver/openapi/markdown_extension.py b/zerver/openapi/markdown_extension.py index 1a6c5d946b..4dc85f88b7 100644 --- a/zerver/openapi/markdown_extension.py +++ b/zerver/openapi/markdown_extension.py @@ -1,16 +1,15 @@ -import re -import json import inspect +import json +import re +from typing import Any, Dict, List, Optional, Pattern, Tuple +import markdown from django.conf import settings - from markdown.extensions import Extension from markdown.preprocessors import Preprocessor -from typing import Any, Dict, Optional, List, Tuple, Pattern -import markdown import zerver.openapi.python_examples -from zerver.openapi.openapi import get_openapi_fixture, openapi_spec, get_openapi_description +from zerver.openapi.openapi import get_openapi_description, get_openapi_fixture, openapi_spec MACRO_REGEXP = re.compile( r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}') diff --git a/zerver/openapi/python_examples.py b/zerver/openapi/python_examples.py index abcd4e864b..885ff60594 100644 --- a/zerver/openapi/python_examples.py +++ b/zerver/openapi/python_examples.py @@ -1,17 +1,15 @@ -from typing import Dict, Any, Optional, Iterable, Callable, Set, List - import json import os import sys from functools import wraps - -from zerver.lib import mdiff -from zerver.openapi.openapi import validate_against_openapi_schema - -from zerver.models import get_realm, get_user +from typing import Any, Callable, Dict, Iterable, List, Optional, Set from zulip import Client +from zerver.lib import mdiff +from zerver.models import get_realm, get_user +from zerver.openapi.openapi import validate_against_openapi_schema + ZULIP_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) TEST_FUNCTIONS: Dict[str, Callable[..., None]] = dict() diff --git a/zerver/openapi/test_curl_examples.py b/zerver/openapi/test_curl_examples.py index 9d1ec803ce..f2e6a268c3 100644 --- a/zerver/openapi/test_curl_examples.py +++ b/zerver/openapi/test_curl_examples.py @@ -1,14 +1,19 @@ import glob +import html import json import shlex import subprocess -import markdown -import html +import markdown from zulip import Client -from zerver.openapi import markdown_extension + from zerver.models import get_realm -from zerver.openapi.curl_param_value_generators import REGISTERED_GENERATOR_FUNCTIONS, CALLED_GENERATOR_FUNCTIONS +from zerver.openapi import markdown_extension +from zerver.openapi.curl_param_value_generators import ( + CALLED_GENERATOR_FUNCTIONS, + REGISTERED_GENERATOR_FUNCTIONS, +) + def test_generated_curl_examples_for_success(client: Client) -> None: authentication_line = f"{client.email}:{client.api_key}" diff --git a/zerver/signals.py b/zerver/signals.py index ddcf2c080b..86a40dfe14 100644 --- a/zerver/signals.py +++ b/zerver/signals.py @@ -3,8 +3,7 @@ from typing import Any, Optional from django.conf import settings from django.contrib.auth.signals import user_logged_in, user_logged_out from django.dispatch import receiver -from django.utils.timezone import \ - get_current_timezone_name as timezone_get_current_timezone_name +from django.utils.timezone import get_current_timezone_name as timezone_get_current_timezone_name from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ @@ -12,8 +11,8 @@ from confirmation.models import one_click_unsubscribe_link from zerver.lib.actions import do_set_zoom_token from zerver.lib.queue import queue_json_publish from zerver.lib.send_email import FromAddress -from zerver.models import UserProfile from zerver.lib.timezone import get_timezone +from zerver.models import UserProfile JUST_CREATED_THRESHOLD = 60 diff --git a/zerver/templatetags/app_filters.py b/zerver/templatetags/app_filters.py index ad5437159c..2c23d10f17 100644 --- a/zerver/templatetags/app_filters.py +++ b/zerver/templatetags/app_filters.py @@ -9,17 +9,17 @@ from django.template import Library, engines from django.utils.safestring import mark_safe from jinja2.exceptions import TemplateNotFound -import zerver.lib.bugdown.fenced_code import zerver.lib.bugdown.api_arguments_table_generator import zerver.lib.bugdown.api_return_values_table_generator -import zerver.openapi.markdown_extension +import zerver.lib.bugdown.fenced_code +import zerver.lib.bugdown.help_emoticon_translations_table +import zerver.lib.bugdown.help_relative_links +import zerver.lib.bugdown.help_settings_links +import zerver.lib.bugdown.include import zerver.lib.bugdown.nested_code_blocks import zerver.lib.bugdown.tabbed_sections -import zerver.lib.bugdown.help_settings_links -import zerver.lib.bugdown.help_relative_links -import zerver.lib.bugdown.help_emoticon_translations_table -import zerver.lib.bugdown.include -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +import zerver.openapi.markdown_extension +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict register = Library() diff --git a/zerver/tests/test_alert_words.py b/zerver/tests/test_alert_words.py index 1ca4d37534..4dd5583f36 100644 --- a/zerver/tests/test_alert_words.py +++ b/zerver/tests/test_alert_words.py @@ -1,28 +1,12 @@ -from zerver.lib.alert_words import ( - alert_words_in_realm, - user_alert_words, -) - -from zerver.lib.actions import ( - do_add_alert_words, - do_remove_alert_words, -) - -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - UserProfile, -) - import ujson +from zerver.lib.actions import do_add_alert_words, do_remove_alert_words +from zerver.lib.alert_words import alert_words_in_realm, user_alert_words +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import UserProfile + + class AlertWordTests(ZulipTestCase): interesting_alert_word_list = ['alert', 'multi-word word', '☃'] diff --git a/zerver/tests/test_archive.py b/zerver/tests/test_archive.py index c42ecbe322..24a333ac94 100644 --- a/zerver/tests/test_archive.py +++ b/zerver/tests/test_archive.py @@ -1,10 +1,15 @@ from django.http import HttpResponse + +from zerver.lib.actions import ( + do_change_stream_web_public, + do_deactivate_stream, + get_web_public_streams, + get_web_public_subs, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.actions import do_change_stream_web_public -from zerver.lib.actions import get_web_public_streams, get_web_public_subs, \ - do_deactivate_stream from zerver.models import get_realm + class GlobalPublicStreamTest(ZulipTestCase): def test_non_existant_stream_id(self) -> None: # Here we use a relatively big number as stream id assuming such an id diff --git a/zerver/tests/test_attachments.py b/zerver/tests/test_attachments.py index 5fefdb9471..7b8ec3b832 100644 --- a/zerver/tests/test_attachments.py +++ b/zerver/tests/test_attachments.py @@ -1,6 +1,5 @@ -from unittest import mock - from typing import Any +from unittest import mock from zerver.lib.attachments import user_attachments from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_audit_log.py b/zerver/tests/test_audit_log.py index 34ed744067..8718eda40a 100644 --- a/zerver/tests/test_audit_log.py +++ b/zerver/tests/test_audit_log.py @@ -1,20 +1,32 @@ +from datetime import timedelta +from typing import Any, Dict + +import ujson +from django.contrib.auth.password_validation import validate_password from django.utils.timezone import now as timezone_now -from zerver.lib.actions import do_create_user, do_deactivate_user, \ - do_activate_user, do_reactivate_user, do_change_password, \ - do_change_user_delivery_email, do_change_avatar_fields, do_change_bot_owner, \ - do_regenerate_api_key, do_change_tos_version, \ - bulk_add_subscriptions, bulk_remove_subscriptions, get_streams_traffic, \ - do_change_user_role, do_deactivate_realm, do_reactivate_realm -from zerver.lib.test_classes import ZulipTestCase -from zerver.models import RealmAuditLog, get_client, get_realm, UserProfile from analytics.models import StreamCount +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_activate_user, + do_change_avatar_fields, + do_change_bot_owner, + do_change_password, + do_change_tos_version, + do_change_user_delivery_email, + do_change_user_role, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_regenerate_api_key, + get_streams_traffic, +) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import RealmAuditLog, UserProfile, get_client, get_realm -from datetime import timedelta -from django.contrib.auth.password_validation import validate_password - -from typing import Any, Dict -import ujson class TestRealmAuditLog(ZulipTestCase): def check_role_count_schema(self, role_counts: Dict[str, Any]) -> None: diff --git a/zerver/tests/test_auth_backends.py b/zerver/tests/test_auth_backends.py index 992cbc27e7..6621cba44b 100644 --- a/zerver/tests/test_auth_backends.py +++ b/zerver/tests/test_auth_backends.py @@ -1,29 +1,39 @@ +import base64 +import copy +import datetime +import json +import re +import time +import urllib +from typing import Any, Callable, Dict, List, Optional, Tuple +from unittest import mock + +import jwt +import ldap +import requests +import responses +import ujson from bs4 import BeautifulSoup from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.contrib.auth import authenticate from django.core import mail -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.test import override_settings -from django_auth_ldap.backend import LDAPSearch, _LDAPUser from django.test.client import RequestFactory -from django.utils.timezone import now as timezone_now -from typing import Any, Callable, Dict, List, Optional, Tuple from django.urls import reverse +from django.utils.timezone import now as timezone_now +from django_auth_ldap.backend import LDAPSearch, _LDAPUser +from onelogin.saml2.auth import OneLogin_Saml2_Auth +from onelogin.saml2.response import OneLogin_Saml2_Response +from social_core.exceptions import AuthFailed, AuthStateForbidden +from social_django.storage import BaseDjangoStorage +from social_django.strategy import DjangoStrategy -import responses - -import ldap -import jwt -from unittest import mock -import re -import datetime -import time -import requests - +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( - do_create_user, do_create_realm, + do_create_user, do_deactivate_realm, do_deactivate_user, do_invite_users, @@ -35,57 +45,92 @@ from zerver.lib.actions import ( from zerver.lib.avatar import avatar_url from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.dev_ldap_directory import generate_dev_ldap_dir -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_is_valid, get_existing_user_errors +from zerver.lib.email_validation import ( + get_existing_user_errors, + get_realm_email_validator, + validate_email_is_valid, +) from zerver.lib.exceptions import RateLimited +from zerver.lib.initial_password import initial_password from zerver.lib.mobile_auth_otp import otp_decrypt_api_key -from zerver.lib.validator import validate_login_email, \ - check_bool, check_dict_only, check_list, check_string, check_int, Validator from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.request import JsonableError from zerver.lib.storage import static_path -from zerver.lib.upload import resize_avatar, MEDIUM_AVATAR_SIZE +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_s3_buckets, + get_test_image_file, + load_subdomain_token, + use_s3_backend, +) +from zerver.lib.upload import MEDIUM_AVATAR_SIZE, resize_avatar from zerver.lib.users import get_all_api_keys from zerver.lib.utils import generate_random_token -from zerver.lib.initial_password import initial_password -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.validator import ( + Validator, + check_bool, + check_dict_only, + check_int, + check_list, + check_string, + validate_login_email, +) +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + MultiuseInvite, + PasswordTooWeakError, + PreregistrationUser, + Realm, + RealmDomain, + UserProfile, + clear_supported_auth_backends_cache, + email_to_username, + get_realm, + get_user_by_delivery_email, ) -from zerver.models import \ - get_realm, email_to_username, CustomProfileField, CustomProfileFieldValue, \ - UserProfile, PreregistrationUser, Realm, RealmDomain, MultiuseInvite, \ - clear_supported_auth_backends_cache, PasswordTooWeakError, get_user_by_delivery_email from zerver.signals import JUST_CREATED_THRESHOLD - -from confirmation.models import Confirmation, create_confirmation_link - -from zproject.backends import ZulipDummyBackend, EmailAuthBackend, AppleAuthBackend, \ - GoogleAuthBackend, ZulipRemoteUserBackend, ZulipLDAPAuthBackend, \ - ZulipLDAPUserPopulator, DevAuthBackend, GitHubAuthBackend, GitLabAuthBackend, ZulipAuthMixin, \ - dev_auth_enabled, password_auth_enabled, github_auth_enabled, gitlab_auth_enabled, \ - apple_auth_enabled, google_auth_enabled, require_email_format_usernames, AUTH_BACKEND_NAME_MAP, \ - ZulipLDAPConfigurationError, ZulipLDAPExceptionNoMatchingLDAPUser, ZulipLDAPExceptionOutsideDomain, \ - ZulipLDAPException, query_ldap, sync_user_from_ldap, SocialAuthMixin, \ - PopulateUserLDAPError, SAMLAuthBackend, saml_auth_enabled, email_belongs_to_ldap, \ - get_external_method_dicts, AzureADAuthBackend, check_password_strength, \ - ZulipLDAPUser, RateLimitedAuthenticationByUsername, ExternalAuthResult, \ - ExternalAuthDataDict - from zerver.views.auth import maybe_send_to_registration +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + AppleAuthBackend, + AzureADAuthBackend, + DevAuthBackend, + EmailAuthBackend, + ExternalAuthDataDict, + ExternalAuthResult, + GitHubAuthBackend, + GitLabAuthBackend, + GoogleAuthBackend, + PopulateUserLDAPError, + RateLimitedAuthenticationByUsername, + SAMLAuthBackend, + SocialAuthMixin, + ZulipAuthMixin, + ZulipDummyBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipLDAPException, + ZulipLDAPExceptionNoMatchingLDAPUser, + ZulipLDAPExceptionOutsideDomain, + ZulipLDAPUser, + ZulipLDAPUserPopulator, + ZulipRemoteUserBackend, + apple_auth_enabled, + check_password_strength, + dev_auth_enabled, + email_belongs_to_ldap, + get_external_method_dicts, + github_auth_enabled, + gitlab_auth_enabled, + google_auth_enabled, + password_auth_enabled, + query_ldap, + require_email_format_usernames, + saml_auth_enabled, + sync_user_from_ldap, +) -from onelogin.saml2.auth import OneLogin_Saml2_Auth -from onelogin.saml2.response import OneLogin_Saml2_Response -from social_core.exceptions import AuthFailed, AuthStateForbidden -from social_django.strategy import DjangoStrategy -from social_django.storage import BaseDjangoStorage - -import base64 -import copy -import json -import urllib -import ujson -from zerver.lib.test_helpers import load_subdomain_token, \ - use_s3_backend, create_s3_buckets, get_test_image_file class AuthBackendTest(ZulipTestCase): def get_username(self, email_to_username: Optional[Callable[[str], str]]=None) -> str: diff --git a/zerver/tests/test_bots.py b/zerver/tests/test_bots.py index a1e1069456..40267ecee7 100644 --- a/zerver/tests/test_bots.py +++ b/zerver/tests/test_bots.py @@ -1,28 +1,38 @@ import filecmp import os -import ujson +from typing import Any, Dict, List, Mapping, Optional +from unittest.mock import MagicMock, patch +import ujson from django.core import mail from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Dict, List, Mapping, Optional +from zulip_bots.custom_exceptions import ConfigValidationError -from zerver.lib.actions import do_change_stream_invite_only, do_deactivate_user, \ - do_set_realm_property -from zerver.lib.bot_config import get_bot_config, ConfigError -from zerver.models import get_realm, get_stream, \ - Realm, UserProfile, get_user, get_bot_services, Service, \ - is_cross_realm_bot_email -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.actions import ( + do_change_stream_invite_only, + do_deactivate_user, + do_set_realm_property, +) +from zerver.lib.bot_config import ConfigError, get_bot_config +from zerver.lib.bot_lib import get_bot_handler +from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, get_test_image_file, queries_captured, tornado_redirected_to_list, ) -from zerver.lib.integrations import EMBEDDED_BOTS, WebhookIntegration -from zerver.lib.bot_lib import get_bot_handler -from zulip_bots.custom_exceptions import ConfigValidationError +from zerver.models import ( + Realm, + Service, + UserProfile, + get_bot_services, + get_realm, + get_stream, + get_user, + is_cross_realm_bot_email, +) # A test validator diff --git a/zerver/tests/test_bugdown.py b/zerver/tests/test_bugdown.py index f3e7efb2b9..b8311d8259 100644 --- a/zerver/tests/test_bugdown.py +++ b/zerver/tests/test_bugdown.py @@ -1,12 +1,19 @@ +import copy +import os +import re +from typing import Any, Dict, List, Optional, Set, Tuple, cast +from unittest import mock + +import ujson from django.conf import settings from django.test import TestCase, override_settings -from zerver.lib import bugdown +from zerver.lib import bugdown, mdiff from zerver.lib.actions import ( - do_set_user_display_setting, - do_remove_realm_emoji, do_add_alert_words, + do_remove_realm_emoji, do_set_realm_property, + do_set_user_display_setting, ) from zerver.lib.alert_words import get_alert_word_automaton from zerver.lib.create_user import create_user @@ -14,42 +21,30 @@ from zerver.lib.emoji import get_emoji_url from zerver.lib.exceptions import BugdownRenderingException from zerver.lib.mention import possible_mentions, possible_user_group_mentions from zerver.lib.message import render_markdown -from zerver.lib.request import ( - JsonableError, -) -from zerver.lib.user_groups import create_user_group -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.request import JsonableError +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow -from zerver.lib import mdiff from zerver.lib.tex import render_tex +from zerver.lib.user_groups import create_user_group from zerver.models import ( - realm_in_local_realm_filters_cache, + MAX_MESSAGE_LENGTH, + Message, + Realm, + RealmEmoji, + RealmFilter, + Stream, + UserGroup, + UserMessage, + UserProfile, flush_per_request_caches, flush_realm_filter, get_client, get_realm, get_stream, realm_filters_for_realm, - MAX_MESSAGE_LENGTH, - Message, - Stream, - Realm, - RealmEmoji, - RealmFilter, - UserMessage, - UserProfile, - UserGroup, + realm_in_local_realm_filters_cache, ) -import copy -from unittest import mock -import os -import ujson -import re - -from typing import cast, Any, Dict, List, Optional, Set, Tuple class FakeMessage: pass diff --git a/zerver/tests/test_cache.py b/zerver/tests/test_cache.py index d9c1a32482..a28455a856 100644 --- a/zerver/tests/test_cache.py +++ b/zerver/tests/test_cache.py @@ -1,16 +1,31 @@ +from typing import Any, Dict, List, Optional +from unittest.mock import Mock, patch + from django.conf import settings -from unittest.mock import Mock, patch -from typing import Any, List, Dict, Optional - from zerver.apps import flush_cache -from zerver.lib.cache import generic_bulk_cached_fetch, user_profile_by_email_cache_key, cache_with_key, \ - validate_cache_key, InvalidCacheKeyException, MEMCACHED_MAX_KEY_LENGTH, get_cache_with_key, \ - NotFoundInCache, cache_set, cache_get, cache_delete, cache_delete_many, cache_get_many, cache_set_many, \ - safe_cache_get_many, safe_cache_set_many +from zerver.lib.cache import ( + MEMCACHED_MAX_KEY_LENGTH, + InvalidCacheKeyException, + NotFoundInCache, + cache_delete, + cache_delete_many, + cache_get, + cache_get_many, + cache_set, + cache_set_many, + cache_with_key, + generic_bulk_cached_fetch, + get_cache_with_key, + safe_cache_get_many, + safe_cache_set_many, + user_profile_by_email_cache_key, + validate_cache_key, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_system_bot, get_user_profile_by_email, UserProfile +from zerver.models import UserProfile, get_system_bot, get_user_profile_by_email + class AppsTest(ZulipTestCase): def test_cache_gets_flushed(self) -> None: diff --git a/zerver/tests/test_camo.py b/zerver/tests/test_camo.py index 8224047a31..3ef7c1e28a 100644 --- a/zerver/tests/test_camo.py +++ b/zerver/tests/test_camo.py @@ -1,5 +1,6 @@ from zerver.lib.test_classes import ZulipTestCase + class CamoURLTest(ZulipTestCase): def test_legacy_camo_url(self) -> None: # Test with valid hex and url pair diff --git a/zerver/tests/test_compatibility.py b/zerver/tests/test_compatibility.py index 6f21f510d2..0b8101d308 100644 --- a/zerver/tests/test_compatibility.py +++ b/zerver/tests/test_compatibility.py @@ -1,7 +1,7 @@ from unittest import mock from zerver.lib.test_classes import ZulipTestCase -from zerver.views.compatibility import find_mobile_os, version_lt, is_outdated_desktop_app +from zerver.views.compatibility import find_mobile_os, is_outdated_desktop_app, version_lt class VersionTest(ZulipTestCase): diff --git a/zerver/tests/test_create_video_call.py b/zerver/tests/test_create_video_call.py index f4c1bbb238..e393b1da2d 100644 --- a/zerver/tests/test_create_video_call.py +++ b/zerver/tests/test_create_video_call.py @@ -1,4 +1,5 @@ import responses + from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py index 6f2cf7d6b8..7889531f4d 100644 --- a/zerver/tests/test_custom_profile_data.py +++ b/zerver/tests/test_custom_profile_data.py @@ -1,17 +1,25 @@ -from typing import Union, List, Dict, Any +from typing import Any, Dict, List, Union +from unittest import mock -from zerver.lib.actions import try_add_realm_custom_profile_field, \ - do_update_user_custom_profile_data_if_changed, do_remove_realm_custom_profile_field, \ - try_reorder_realm_custom_profile_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import queries_captured -from zerver.lib.bugdown import convert as bugdown_convert -from zerver.models import CustomProfileField, \ - custom_profile_fields_for_realm, CustomProfileFieldValue, get_realm -from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS import ujson -from unittest import mock +from zerver.lib.actions import ( + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_reorder_realm_custom_profile_fields, +) +from zerver.lib.bugdown import convert as bugdown_convert +from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + custom_profile_fields_for_realm, + get_realm, +) + class CustomProfileFieldTestCase(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_decorators.py b/zerver/tests/test_decorators.py index f20da1ea75..bbc82ccc3b 100644 --- a/zerver/tests/test_decorators.py +++ b/zerver/tests/test_decorators.py @@ -1,57 +1,81 @@ import base64 -from unittest import mock -import re import os +import re from collections import defaultdict - from typing import Any, Dict, Iterable, List, Optional, Tuple +from unittest import mock -from django.test import TestCase -from django.http import HttpResponse, HttpRequest +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.test import TestCase -from zerver.forms import OurAuthenticationForm -from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, \ - do_reactivate_user, do_reactivate_realm, do_set_realm_property -from zerver.lib.exceptions import JsonableError, InvalidAPIKeyError, InvalidAPIKeyFormatError -from zerver.lib.initial_password import initial_password -from zerver.lib.test_helpers import ( - HostRequestMock, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.response import json_response, json_success -from zerver.lib.users import get_api_key -from zerver.lib.user_agent import parse_user_agent -from zerver.lib.utils import generate_api_key, has_api_key_format -from zerver.lib.request import \ - REQ, has_request_variables, RequestVariableMissingError, \ - RequestVariableConversionError, RequestConfusingParmsError -from zerver.lib.webhooks.common import UnexpectedWebhookEventType from zerver.decorator import ( api_key_only_webhook_view, + authenticate_notify, authenticated_json_view, authenticated_rest_api_view, authenticated_uploads_api_view, - authenticate_notify, cachify, - get_client_name, internal_notify_view, is_local_addr, - rate_limit, validate_api_key, + cachify, + get_client_name, + internal_notify_view, + is_local_addr, + rate_limit, return_success_on_head_request, + validate_api_key, zulip_login_required, ) -from zerver.lib.cache import ignore_unhashable_lru_cache, dict_to_items_tuple, items_tuple_to_dict +from zerver.forms import OurAuthenticationForm +from zerver.lib.actions import ( + do_deactivate_realm, + do_deactivate_user, + do_reactivate_realm, + do_reactivate_user, + do_set_realm_property, +) +from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict +from zerver.lib.exceptions import InvalidAPIKeyError, InvalidAPIKeyFormatError, JsonableError +from zerver.lib.initial_password import initial_password +from zerver.lib.request import ( + REQ, + RequestConfusingParmsError, + RequestVariableConversionError, + RequestVariableMissingError, + has_request_variables, +) +from zerver.lib.response import json_response, json_success +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.user_agent import parse_user_agent +from zerver.lib.users import get_api_key +from zerver.lib.utils import generate_api_key, has_api_key_format from zerver.lib.validator import ( - check_string, check_dict, check_dict_only, check_bool, check_float, check_int, check_list, Validator, - check_variable_type, equals, check_none_or, check_url, check_short_string, - check_string_fixed_length, check_capped_string, check_color, to_non_negative_int, - check_string_or_int_list, check_string_or_int, check_int_in, check_string_in, + Validator, + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_short_string, + check_string, + check_string_fixed_length, + check_string_in, + check_string_or_int, + check_string_or_int_list, + check_url, + check_variable_type, + equals, + to_non_negative_int, to_positive_or_allowed_int, ) -from zerver.models import \ - get_realm, get_user, UserProfile, Realm +from zerver.lib.webhooks.common import UnexpectedWebhookEventType +from zerver.models import Realm, UserProfile, get_realm, get_user -import ujson class DecoratorTestCase(TestCase): def test_get_client_name(self) -> None: diff --git a/zerver/tests/test_digest.py b/zerver/tests/test_digest.py index 6f634d6765..cc3d3fb124 100644 --- a/zerver/tests/test_digest.py +++ b/zerver/tests/test_digest.py @@ -1,20 +1,34 @@ import datetime -from unittest import mock import time from typing import List +from unittest import mock from django.test import override_settings from django.utils.timezone import now as timezone_now from confirmation.models import one_click_unsubscribe_link from zerver.lib.actions import do_create_user -from zerver.lib.digest import gather_new_streams, handle_digest_email, enqueue_emails, \ - exclude_subscription_modified_streams +from zerver.lib.digest import ( + enqueue_emails, + exclude_subscription_modified_streams, + gather_new_streams, + handle_digest_email, +) from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured -from zerver.models import get_client, get_realm, flush_per_request_caches, \ - Realm, Message, UserActivity, UserProfile, RealmAuditLog, get_stream +from zerver.models import ( + Message, + Realm, + RealmAuditLog, + UserActivity, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, +) + class TestDigestEmailMessages(ZulipTestCase): diff --git a/zerver/tests/test_docs.py b/zerver/tests/test_docs.py index ee86e4041f..e6c401510f 100644 --- a/zerver/tests/test_docs.py +++ b/zerver/tests/test_docs.py @@ -1,12 +1,12 @@ import os -import ujson +from typing import Any, Dict, List from unittest import mock from urllib.parse import urlsplit +import ujson from django.conf import settings -from django.test import TestCase, override_settings from django.http import HttpResponse -from typing import Any, Dict, List +from django.test import TestCase, override_settings from zerver.lib.integrations import INTEGRATIONS from zerver.lib.test_classes import ZulipTestCase @@ -14,9 +14,8 @@ from zerver.lib.test_helpers import HostRequestMock from zerver.lib.test_runner import slow from zerver.lib.utils import split_by from zerver.models import Realm, get_realm -from zerver.views.documentation import ( - add_api_uri_context, -) +from zerver.views.documentation import add_api_uri_context + class DocPageTest(ZulipTestCase): def get_doc(self, url: str, subdomain: str) -> HttpResponse: diff --git a/zerver/tests/test_email_change.py b/zerver/tests/test_email_change.py index 6b8685774c..651dee7ed9 100644 --- a/zerver/tests/test_email_change.py +++ b/zerver/tests/test_email_change.py @@ -1,17 +1,22 @@ import datetime -from email.utils import parseaddr import re +from email.utils import parseaddr from django.core import mail from django.utils.timezone import now -from confirmation.models import Confirmation, generate_key, confirmation_url -from zerver.lib.actions import do_start_email_change_process, do_set_realm_property -from zerver.lib.test_classes import ( - ZulipTestCase, +from confirmation.models import Confirmation, confirmation_url, generate_key +from zerver.lib.actions import do_set_realm_property, do_start_email_change_process +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ( + EmailChangeStatus, + Realm, + UserProfile, + get_realm, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, ) -from zerver.models import get_user_by_delivery_email, EmailChangeStatus, get_realm, \ - Realm, UserProfile, get_user, get_user_profile_by_id class EmailChangeTestCase(ZulipTestCase): diff --git a/zerver/tests/test_email_log.py b/zerver/tests/test_email_log.py index a720edf058..b21f9fcb66 100644 --- a/zerver/tests/test_email_log.py +++ b/zerver/tests/test_email_log.py @@ -1,10 +1,12 @@ import os from unittest import mock + from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zproject.email_backends import get_forward_address + class EmailLogTest(ZulipTestCase): def test_generate_and_clear_email_log(self) -> None: with self.settings(EMAIL_BACKEND='zproject.email_backends.EmailLogBackEnd'), \ diff --git a/zerver/tests/test_email_mirror.py b/zerver/tests/test_email_mirror.py index c4205ff9bb..d516807b69 100644 --- a/zerver/tests/test_email_mirror.py +++ b/zerver/tests/test_email_mirror.py @@ -1,63 +1,50 @@ +import os import subprocess +from email import message_from_string +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from typing import Any, Callable, Dict, Mapping, Optional +from unittest import mock +import ujson +from django.conf import settings from django.http import HttpResponse -from zerver.lib.test_helpers import ( - most_recent_message, - most_recent_usermessage, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - get_display_recipient, - get_realm, - get_stream, - get_system_bot, - MissedMessageEmailAddress, - Recipient, - UserProfile, -) - -from zerver.lib.actions import ensure_stream, do_deactivate_realm, do_deactivate_user - +from zerver.lib.actions import do_deactivate_realm, do_deactivate_user, ensure_stream from zerver.lib.email_mirror import ( - process_message, - process_missed_message, + ZulipEmailForwardError, create_missed_message_address, + filter_footer, get_missed_message_token_from_address, - strip_from_subject, is_forwarded, is_missed_message_address, - filter_footer, log_and_report, + process_message, + process_missed_message, redact_email_address, - ZulipEmailForwardError, + strip_from_subject, ) - from zerver.lib.email_mirror_helpers import ( decode_email_address, encode_email_address, get_email_gateway_message_string_from_address, ) - from zerver.lib.email_notifications import convert_html_to_markdown from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, most_recent_usermessage +from zerver.models import ( + MissedMessageEmailAddress, + Recipient, + UserProfile, + get_display_recipient, + get_realm, + get_stream, + get_system_bot, +) from zerver.worker.queue_processors import MirrorWorker -from email import message_from_string -from email.mime.text import MIMEText -from email.mime.image import MIMEImage -from email.mime.multipart import MIMEMultipart - -import ujson -from unittest import mock -import os -from django.conf import settings - -from typing import Any, Callable, Dict, Mapping, Optional class TestEncodeDecode(ZulipTestCase): def _assert_options(self, options: Dict[str, bool], show_sender: bool=False, diff --git a/zerver/tests/test_email_notifications.py b/zerver/tests/test_email_notifications.py index b8720dd380..bbb134c7cd 100644 --- a/zerver/tests/test_email_notifications.py +++ b/zerver/tests/test_email_notifications.py @@ -1,27 +1,27 @@ -import ldap import random import re -import ujson +from email.utils import formataddr +from typing import List, Optional +from unittest.mock import patch +import ldap +import ujson from django.conf import settings from django.core import mail from django.test import override_settings from django_auth_ldap.config import LDAPSearch -from email.utils import formataddr -from unittest.mock import patch -from typing import List, Optional -from zerver.lib.email_notifications import fix_emojis, handle_missedmessage_emails, \ - enqueue_welcome_emails, relative_to_full_url from zerver.lib.actions import do_change_notification_settings, do_change_user_role -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.send_email import FromAddress, send_custom_email -from zerver.models import ( - get_realm, - get_stream, - UserProfile, - ScheduledEmail, +from zerver.lib.email_notifications import ( + enqueue_welcome_emails, + fix_emojis, + handle_missedmessage_emails, + relative_to_full_url, ) +from zerver.lib.send_email import FromAddress, send_custom_email +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import ScheduledEmail, UserProfile, get_realm, get_stream + class TestCustomEmails(ZulipTestCase): def test_send_custom_email_argument(self) -> None: diff --git a/zerver/tests/test_embedded_bot_system.py b/zerver/tests/test_embedded_bot_system.py index f70fdb5f8d..768237544d 100644 --- a/zerver/tests/test_embedded_bot_system.py +++ b/zerver/tests/test_embedded_bot_system.py @@ -1,13 +1,17 @@ from unittest.mock import patch +import ujson + from zerver.lib.bot_lib import EmbeddedBotQuitException from zerver.lib.test_classes import ZulipTestCase from zerver.models import ( - UserProfile, get_display_recipient, - get_service_profile, get_user, get_realm, + UserProfile, + get_display_recipient, + get_realm, + get_service_profile, + get_user, ) -import ujson class TestEmbeddedBotMessaging(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index f80c8b1eed..5a9cca4e83 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -1,18 +1,24 @@ -from unittest import mock import time -import ujson - -from django.http import HttpRequest, HttpResponse from typing import Any, Callable, Dict, Tuple +from unittest import mock -from zerver.lib.actions import do_mute_topic, do_change_subscription_property +import ujson +from django.http import HttpRequest, HttpResponse + +from zerver.lib.actions import do_change_subscription_property, do_mute_topic from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import POSTRequestMock from zerver.models import Recipient, Stream, Subscription, UserProfile, get_stream -from zerver.tornado.event_queue import maybe_enqueue_notifications, \ - allocate_client_descriptor, ClientDescriptor, \ - get_client_descriptor, missedmessage_hook, persistent_queue_filename -from zerver.tornado.views import get_events, cleanup_event_queue +from zerver.tornado.event_queue import ( + ClientDescriptor, + allocate_client_descriptor, + get_client_descriptor, + maybe_enqueue_notifications, + missedmessage_hook, + persistent_queue_filename, +) +from zerver.tornado.views import cleanup_event_queue, get_events + class MissedMessageNotificationsTest(ZulipTestCase): """Tests the logic for when missed-message notifications diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 85f36b5a9c..f2a817df4f 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -1,29 +1,26 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import Any, Callable, Dict, List, Optional, Set, Tuple import copy import os import shutil import sys +import time +from io import StringIO +from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from unittest import mock +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now -from io import StringIO - -from zerver.models import ( - get_client, get_stream, get_realm, get_system_bot, - Message, RealmDomain, Recipient, UserMessage, UserPresence, UserProfile, - Realm, Subscription, Stream, flush_per_request_caches, UserGroup, Service, - Attachment, PreregistrationUser, get_user_by_delivery_email, MultiuseInvite, - RealmAuditLog, -) from zerver.lib.actions import ( - try_update_realm_custom_profile_field, + bulk_add_members_to_user_group, bulk_add_subscriptions, bulk_remove_subscriptions, check_add_realm_emoji, + check_add_user_group, + check_delete_user_group, check_send_message, check_send_typing_notification, do_add_alert_words, @@ -44,7 +41,6 @@ from zerver.lib.actions import ( do_change_full_name, do_change_icon_source, do_change_logo_source, - do_change_user_role, do_change_notification_settings, do_change_plan_type, do_change_realm_domain, @@ -53,9 +49,10 @@ from zerver.lib.actions import ( do_change_stream_post_policy, do_change_subscription_property, do_change_user_delivery_email, - do_create_user, + do_change_user_role, do_create_default_stream_group, do_create_multiuse_invite_link, + do_create_user, do_deactivate_stream, do_deactivate_user, do_delete_messages, @@ -78,10 +75,10 @@ from zerver.lib.actions import ( do_revoke_user_invite, do_set_realm_authentication_methods, do_set_realm_message_editing, - do_set_realm_property, - do_set_user_display_setting, do_set_realm_notifications_stream, + do_set_realm_property, do_set_realm_signup_notifications_stream, + do_set_user_display_setting, do_set_zoom_token, do_unmute_topic, do_update_embedded_data, @@ -89,18 +86,16 @@ from zerver.lib.actions import ( do_update_message_flags, do_update_outgoing_webhook_service, do_update_pointer, + do_update_user_custom_profile_data_if_changed, + do_update_user_group_description, + do_update_user_group_name, do_update_user_presence, do_update_user_status, log_event, lookup_default_stream_groups, notify_realm_custom_profile_fields, - check_add_user_group, - do_update_user_group_name, - do_update_user_group_description, - bulk_add_members_to_user_group, remove_members_from_user_group, - check_delete_user_group, - do_update_user_custom_profile_data_if_changed, + try_update_realm_custom_profile_field, ) from zerver.lib.bugdown import MentionData from zerver.lib.events import ( @@ -110,36 +105,65 @@ from zerver.lib.events import ( post_process_state, ) from zerver.lib.message import ( + MessageDict, + UnreadMessagesResult, aggregate_unread_data, apply_unread_message_event, get_raw_unread_data, render_markdown, - MessageDict, - UnreadMessagesResult, ) -from zerver.lib.test_helpers import POSTRequestMock, get_subscription, \ - get_test_image_file, stub_event_queue_user_events, queries_captured, \ - create_dummy_file, stdout_suppressed, reset_emails_in_zulip_realm -from zerver.lib.test_classes import ( - ZulipTestCase, +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + POSTRequestMock, + create_dummy_file, + get_subscription, + get_test_image_file, + queries_captured, + reset_emails_in_zulip_realm, + stdout_suppressed, + stub_event_queue_user_events, ) from zerver.lib.test_runner import slow -from zerver.lib.topic import ( - ORIG_TOPIC, - TOPIC_NAME, - TOPIC_LINKS, -) -from zerver.lib.topic_mutes import ( - add_topic_mute, -) -from zerver.lib.validator import ( - check_bool, check_dict, check_dict_only, check_float, check_int, check_list, check_string, - equals, check_none_or, Validator, check_url, check_int_in, -) +from zerver.lib.topic import ORIG_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.topic_mutes import add_topic_mute from zerver.lib.users import get_api_key - -from zerver.views.events_register import _default_all_public_streams, _default_narrow - +from zerver.lib.validator import ( + Validator, + check_bool, + check_dict, + check_dict_only, + check_float, + check_int, + check_int_in, + check_list, + check_none_or, + check_string, + check_url, + equals, +) +from zerver.models import ( + Attachment, + Message, + MultiuseInvite, + PreregistrationUser, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserGroup, + UserMessage, + UserPresence, + UserProfile, + flush_per_request_caches, + get_client, + get_realm, + get_stream, + get_system_bot, + get_user_by_delivery_email, +) from zerver.tornado.event_queue import ( allocate_client_descriptor, clear_client_event_queues_for_testing, @@ -147,10 +171,7 @@ from zerver.tornado.event_queue import ( process_message_event, ) from zerver.tornado.views import get_events - -from unittest import mock -import time -import ujson +from zerver.views.events_register import _default_all_public_streams, _default_narrow class LogEventsTest(ZulipTestCase): diff --git a/zerver/tests/test_external.py b/zerver/tests/test_external.py index 38a1886f34..41273b6439 100644 --- a/zerver/tests/test_external.py +++ b/zerver/tests/test_external.py @@ -1,28 +1,22 @@ +import time +from unittest import mock + +import DNS from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpResponse from zerver.forms import email_is_not_mit_mailing_list - from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedUser, RateLimiterLockingException, + add_ratelimit_rule, + remove_ratelimit_rule, ) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.zephyr import compute_mit_user_fullname +from zerver.models import UserProfile -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - UserProfile, -) - -import DNS -from unittest import mock -import time class MITNameTest(ZulipTestCase): def test_valid_hesiod(self) -> None: diff --git a/zerver/tests/test_gitter_importer.py b/zerver/tests/test_gitter_importer.py index a50ba1981d..89c217844e 100644 --- a/zerver/tests/test_gitter_importer.py +++ b/zerver/tests/test_gitter_importer.py @@ -1,24 +1,15 @@ -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - get_realm, - UserProfile, - Message, -) -from zerver.data_import.gitter import ( - do_convert_data, - get_usermentions, -) - -import ujson import logging import os -from unittest import mock from typing import Any +from unittest import mock + +import ujson + +from zerver.data_import.gitter import do_convert_data, get_usermentions +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, UserProfile, get_realm + class GitterImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_hipchat_importer.py b/zerver/tests/test_hipchat_importer.py index 709e2a2191..55657729f1 100644 --- a/zerver/tests/test_hipchat_importer.py +++ b/zerver/tests/test_hipchat_importer.py @@ -1,18 +1,10 @@ -from zerver.data_import.hipchat import ( - get_hipchat_sender_id, -) -from zerver.data_import.hipchat_user import ( - UserHandler, -) -from zerver.data_import.sequencer import ( - IdMapper, -) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) from typing import Any, Dict +from zerver.data_import.hipchat import get_hipchat_sender_id +from zerver.data_import.hipchat_user import UserHandler +from zerver.data_import.sequencer import IdMapper +from zerver.lib.test_classes import ZulipTestCase + class HipChatImporter(ZulipTestCase): def test_sender_ids(self) -> None: diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index b1fc3b62b5..e1b00d40c4 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -1,34 +1,38 @@ import calendar +import urllib from datetime import timedelta +from typing import Any, Dict +from unittest.mock import patch + import lxml.html import ujson - from django.conf import settings from django.http import HttpResponse from django.utils.timezone import now as timezone_now -from unittest.mock import patch -import urllib -from typing import Any, Dict -from zerver.lib.actions import ( - do_create_user, do_change_logo_source, -) + +from corporate.models import Customer, CustomerPlan +from zerver.lib.actions import do_change_logo_source, do_create_user from zerver.lib.events import add_realm_logo_fields -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - queries_captured, get_user_messages, -) from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_user_messages, queries_captured from zerver.lib.test_runner import slow from zerver.lib.users import compute_show_invites_and_add_streams from zerver.models import ( - get_realm, get_stream, get_user, UserProfile, - flush_per_request_caches, DefaultStream, Realm, - get_system_bot, UserActivity, + DefaultStream, + Realm, + UserActivity, + UserProfile, + flush_per_request_caches, + get_realm, + get_stream, + get_system_bot, + get_user, ) from zerver.views.home import compute_navbar_logo_url, get_furthest_read_time -from corporate.models import Customer, CustomerPlan from zerver.worker.queue_processors import UserActivityWorker + class HomeTest(ZulipTestCase): def test_home(self) -> None: diff --git a/zerver/tests/test_hotspots.py b/zerver/tests/test_hotspots.py index c516307d04..59123353b4 100644 --- a/zerver/tests/test_hotspots.py +++ b/zerver/tests/test_hotspots.py @@ -1,9 +1,10 @@ -from zerver.lib.actions import do_mark_hotspot_as_read, do_create_user +import ujson + +from zerver.lib.actions import do_create_user, do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS, get_next_hotspots from zerver.lib.test_classes import ZulipTestCase -from zerver.models import UserProfile, UserHotspot, get_realm +from zerver.models import UserHotspot, UserProfile, get_realm -import ujson # Splitting this out, since I imagine this will eventually have most of the # complicated hotspots logic. diff --git a/zerver/tests/test_i18n.py b/zerver/tests/test_i18n.py index 6c550da043..804998b3fd 100644 --- a/zerver/tests/test_i18n.py +++ b/zerver/tests/test_i18n.py @@ -1,20 +1,19 @@ +from http.cookies import SimpleCookie from typing import Any - from unittest import mock + import ujson +from django.conf import settings +from django.core import mail +from django.http import HttpResponse from django.test import TestCase from django.utils import translation -from django.conf import settings -from django.http import HttpResponse -from django.core import mail -from http.cookies import SimpleCookie -from zerver.models import get_realm_stream -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.management.commands import makemessages from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.test_classes import ZulipTestCase +from zerver.management.commands import makemessages +from zerver.models import get_realm_stream + class EmailTranslationTestCase(ZulipTestCase): def test_email_translation(self) -> None: diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 809c2b345c..0fdacd442a 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -1,95 +1,68 @@ -from django.conf import settings - import os -import ujson - +from typing import Any, Callable, Dict, FrozenSet, List, Optional, Set, Tuple from unittest.mock import patch -from typing import Any, Dict, List, Set, Optional, Tuple, Callable, \ - FrozenSet + +import ujson +from django.conf import settings from django.db.models import Q from django.utils.timezone import now as timezone_now -from zerver.lib.export import ( - do_export_realm, - export_usermessages_batch, - do_export_user, -) -from zerver.lib.import_realm import ( - do_import_realm, - get_incoming_message_ids, -) -from zerver.lib.avatar_hash import ( - user_avatar_path, -) -from zerver.lib.upload import ( - claim_attachment, - upload_message_file, - upload_emoji_image, - upload_avatar_image, -) from zerver.lib import upload - -from zerver.lib.utils import ( - query_chunker, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, - use_s3_backend, - create_s3_buckets, -) - -from zerver.lib.topic_mutes import ( - add_topic_mute, -) -from zerver.lib.bot_lib import ( - StateHandler, -) -from zerver.lib.bot_config import ( - set_bot_config, -) from zerver.lib.actions import ( - do_create_user, do_add_reaction, do_change_icon_source, do_change_logo_source, - do_update_user_presence, do_change_plan_type, + do_create_user, + do_update_user_presence, ) +from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.bot_config import set_bot_config +from zerver.lib.bot_lib import StateHandler +from zerver.lib.export import do_export_realm, do_export_user, export_usermessages_batch +from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids from zerver.lib.streams import create_stream_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, use_s3_backend from zerver.lib.test_runner import slow - +from zerver.lib.topic_mutes import add_topic_mute +from zerver.lib.upload import ( + claim_attachment, + upload_avatar_image, + upload_emoji_image, + upload_message_file, +) +from zerver.lib.utils import query_chunker from zerver.models import ( - Message, - Realm, - Stream, - UserProfile, - Subscription, Attachment, - RealmEmoji, - Reaction, - Recipient, - UserMessage, + BotConfigData, + BotStorageData, CustomProfileField, CustomProfileFieldValue, - RealmAuditLog, Huddle, - UserHotspot, + Message, MutedTopic, + Reaction, + Realm, + RealmAuditLog, + RealmEmoji, + Recipient, + Stream, + Subscription, UserGroup, UserGroupMembership, + UserHotspot, + UserMessage, UserPresence, - BotStorageData, - BotConfigData, + UserProfile, get_active_streams, get_client, + get_huddle_hash, get_realm, get_stream, - get_huddle_hash, ) + class QueryUtilTest(ZulipTestCase): def _create_messages(self) -> None: for name in ['cordelia', 'hamlet', 'iago']: diff --git a/zerver/tests/test_integrations.py b/zerver/tests/test_integrations.py index ae4b0e136a..ff0bb2b7d4 100644 --- a/zerver/tests/test_integrations.py +++ b/zerver/tests/test_integrations.py @@ -1,7 +1,15 @@ -from zerver.lib.test_classes import ZulipTestCase from zerver.lib.integrations import ( - split_fixture_path, get_fixture_and_image_paths, INTEGRATIONS, ScreenshotConfig, WebhookIntegration, - DOC_SCREENSHOT_CONFIG, WEBHOOK_INTEGRATIONS, NO_SCREENSHOT_WEBHOOKS) + DOC_SCREENSHOT_CONFIG, + INTEGRATIONS, + NO_SCREENSHOT_WEBHOOKS, + WEBHOOK_INTEGRATIONS, + ScreenshotConfig, + WebhookIntegration, + get_fixture_and_image_paths, + split_fixture_path, +) +from zerver.lib.test_classes import ZulipTestCase + class IntegrationsTestCase(ZulipTestCase): diff --git a/zerver/tests/test_integrations_dev_panel.py b/zerver/tests/test_integrations_dev_panel.py index 3a6dd4528f..7c7af870fa 100644 --- a/zerver/tests/test_integrations_dev_panel.py +++ b/zerver/tests/test_integrations_dev_panel.py @@ -1,7 +1,10 @@ -import ujson from unittest.mock import MagicMock, patch + +import ujson + from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_user, get_realm, Message, Stream +from zerver.models import Message, Stream, get_realm, get_user + class TestIntegrationsDevPanel(ZulipTestCase): diff --git a/zerver/tests/test_internet.py b/zerver/tests/test_internet.py index 0fee3306e5..6172c6f4c2 100644 --- a/zerver/tests/test_internet.py +++ b/zerver/tests/test_internet.py @@ -1,7 +1,8 @@ +import requests +import responses + from zerver.lib.test_classes import ZulipTestCase -import responses -import requests class ResponsesTest(ZulipTestCase): def test_responses(self) -> None: diff --git a/zerver/tests/test_legacy_subject.py b/zerver/tests/test_legacy_subject.py index 436a9de452..90702c7d69 100644 --- a/zerver/tests/test_legacy_subject.py +++ b/zerver/tests/test_legacy_subject.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class LegacySubjectTest(ZulipTestCase): def test_legacy_subject(self) -> None: diff --git a/zerver/tests/test_link_embed.py b/zerver/tests/test_link_embed.py index 14a1204bc5..c6b19a126a 100644 --- a/zerver/tests/test_link_embed.py +++ b/zerver/tests/test_link_embed.py @@ -1,22 +1,20 @@ -from unittest import mock -import ujson from typing import Any, Callable, Dict, Optional -from requests.exceptions import ConnectionError +from unittest import mock + +import ujson from django.test import override_settings from django.utils.html import escape +from requests.exceptions import ConnectionError -from zerver.models import Message, Realm, UserProfile from zerver.lib.actions import queue_json_publish +from zerver.lib.cache import NotFoundInCache, cache_set, preview_url_cache_key from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import MockPythonResponse -from zerver.worker.queue_processors import FetchLinksEmbedData -from zerver.lib.url_preview.preview import ( - get_link_embed_data, link_embed_data_from_cache) from zerver.lib.url_preview.oembed import get_oembed_data, strip_cdata -from zerver.lib.url_preview.parsers import ( - OpenGraphParser, GenericParser) -from zerver.lib.cache import cache_set, NotFoundInCache, preview_url_cache_key - +from zerver.lib.url_preview.parsers import GenericParser, OpenGraphParser +from zerver.lib.url_preview.preview import get_link_embed_data, link_embed_data_from_cache +from zerver.models import Message, Realm, UserProfile +from zerver.worker.queue_processors import FetchLinksEmbedData TEST_CACHES = { 'default': { diff --git a/zerver/tests/test_logging_handlers.py b/zerver/tests/test_logging_handlers.py index fd90ee3afd..d3a266b7ac 100644 --- a/zerver/tests/test_logging_handlers.py +++ b/zerver/tests/test_logging_handlers.py @@ -1,18 +1,18 @@ import logging import sys +from functools import wraps +from types import TracebackType +from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type +from unittest.mock import MagicMock, patch from django.conf import settings from django.contrib.auth.models import AnonymousUser from django.http import HttpRequest from django.test import TestCase from django.utils.log import AdminEmailHandler -from functools import wraps -from unittest.mock import MagicMock, patch -from types import TracebackType -from typing import Any, Callable, Dict, Iterator, NoReturn, Optional, Tuple, Type -from zerver.lib.types import ViewFuncT from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.types import ViewFuncT from zerver.logging_handlers import AdminNotifyHandler captured_request: Optional[HttpRequest] = None diff --git a/zerver/tests/test_management_commands.py b/zerver/tests/test_management_commands.py index cb47a55667..f0d9d7d26f 100644 --- a/zerver/tests/test_management_commands.py +++ b/zerver/tests/test_management_commands.py @@ -3,24 +3,32 @@ import os import re from datetime import timedelta from email.utils import parseaddr +from typing import Any, Dict, List, Optional from unittest import mock -from unittest.mock import MagicMock, patch, call -from typing import List, Dict, Any, Optional +from unittest.mock import MagicMock, call, patch from django.conf import settings from django.core.management import call_command from django.test import TestCase, override_settings -from zerver.lib.actions import do_create_user, do_add_reaction -from zerver.lib.management import ZulipBaseCommand, CommandError, check_config -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import stdout_suppressed -from zerver.lib.test_runner import slow -from zerver.models import Recipient, get_user_profile_by_email, get_stream from django.utils.timezone import now as timezone_now -from zerver.lib.test_helpers import most_recent_message -from zerver.models import get_realm, UserProfile, Realm, Reaction, Message from confirmation.models import RealmCreationKey, generate_realm_creation_url +from zerver.lib.actions import do_add_reaction, do_create_user +from zerver.lib.management import CommandError, ZulipBaseCommand, check_config +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import most_recent_message, stdout_suppressed +from zerver.lib.test_runner import slow +from zerver.models import ( + Message, + Reaction, + Realm, + Recipient, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, +) + class TestCheckConfig(ZulipTestCase): def test_check_config(self) -> None: diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py index ff0978b3cd..f98784b4d0 100644 --- a/zerver/tests/test_mattermost_importer.py +++ b/zerver/tests/test_mattermost_importer.py @@ -1,27 +1,34 @@ -import os -import ujson import filecmp import logging +import os +from typing import Any, Dict, List -from typing import Dict, Any, List +import ujson -from zerver.lib.emoji import name_to_codepoint -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.data_import.mattermost_user import UserHandler -from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \ - create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \ - convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \ - build_reactions, do_convert_data, convert_huddle_data, \ - generate_huddle_name -from zerver.data_import.sequencer import IdMapper from zerver.data_import.import_util import SubscriberHandler -from zerver.models import Reaction, UserProfile, Message, get_realm, get_user, Recipient +from zerver.data_import.mattermost import ( + build_reactions, + check_user_in_team, + convert_channel_data, + convert_huddle_data, + convert_user_data, + create_username_to_user_mapping, + do_convert_data, + generate_huddle_name, + get_mentioned_user_ids, + label_mirror_dummy_users, + mattermost_data_file_to_dict, + process_user, + reset_mirror_dummy_users, + write_emoticon_data, +) +from zerver.data_import.mattermost_user import UserHandler +from zerver.data_import.sequencer import IdMapper +from zerver.lib.emoji import name_to_codepoint +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, Reaction, Recipient, UserProfile, get_realm, get_user + class MatterMostImporter(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_message_edit_notifications.py b/zerver/tests/test_message_edit_notifications.py index c2f882b2a7..00800c1e78 100644 --- a/zerver/tests/test_message_edit_notifications.py +++ b/zerver/tests/test_message_edit_notifications.py @@ -1,25 +1,13 @@ from typing import Any, Dict, Mapping, Union - from unittest import mock from django.utils.timezone import now as timezone_now -from zerver.lib.actions import ( - get_client, -) +from zerver.lib.actions import get_client +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Subscription, UserPresence +from zerver.tornado.event_queue import maybe_enqueue_notifications -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import ( - Subscription, - UserPresence, -) - -from zerver.tornado.event_queue import ( - maybe_enqueue_notifications, -) class EditMessageSideEffectsTest(ZulipTestCase): def _assert_update_does_not_notify_anybody(self, message_id: int, content: str) -> None: diff --git a/zerver/tests/test_messages.py b/zerver/tests/test_messages.py index 6a110fd8b8..bfec18fcbe 100644 --- a/zerver/tests/test_messages.py +++ b/zerver/tests/test_messages.py @@ -1,15 +1,22 @@ +import datetime +import time +from collections import namedtuple +from operator import itemgetter +from typing import Any, Dict, List, Set, Tuple, Union +from unittest import mock + +import ujson +from django.conf import settings from django.db import IntegrityError from django.db.models import Q -from django.conf import settings from django.http import HttpResponse from django.test import TestCase, override_settings from django.utils.timezone import now as timezone_now -from zerver.lib import bugdown +from analytics.lib.counts import COUNT_STATS +from analytics.models import RealmCount from zerver.decorator import JsonableError -from zerver.lib.test_runner import slow -from zerver.lib.addressee import Addressee - +from zerver.lib import bugdown from zerver.lib.actions import ( check_message, check_send_stream_message, @@ -22,8 +29,8 @@ from zerver.lib.actions import ( do_create_user, do_deactivate_user, do_send_messages, - do_update_message, do_set_realm_property, + do_update_message, extract_private_recipients, extract_stream_indicator, gather_subscriptions_helper, @@ -40,18 +47,9 @@ from zerver.lib.actions import ( internal_send_stream_message_by_name, send_rate_limited_pm_notification_to_bot_owner, ) - -from zerver.lib.cache import ( - cache_delete, - get_stream_cache_key, - to_dict_cache_key_id, -) - - -from zerver.lib.create_user import ( - create_user_profile, -) - +from zerver.lib.addressee import Addressee +from zerver.lib.cache import cache_delete, get_stream_cache_key, to_dict_cache_key_id +from zerver.lib.create_user import create_user_profile from zerver.lib.message import ( MessageDict, bulk_access_messages, @@ -64,7 +62,13 @@ from zerver.lib.message import ( sew_messages_and_reactions, update_first_visible_message_id, ) - +from zerver.lib.soft_deactivation import ( + add_missing_messages, + do_soft_activate_users, + do_soft_deactivate_users, + reactivate_user_if_soft_deactivated, +) +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( get_subscription, get_user_messages, @@ -75,55 +79,42 @@ from zerver.lib.test_helpers import ( queries_captured, reset_emails_in_zulip_realm, ) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.lib.topic import ( - LEGACY_PREV_TOPIC, - DB_TOPIC_NAME, - TOPIC_LINKS, - TOPIC_NAME, -) - -from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient -from zerver.lib.soft_deactivation import ( - add_missing_messages, - do_soft_activate_users, - do_soft_deactivate_users, - reactivate_user_if_soft_deactivated, -) - -from zerver.models import ( - MAX_MESSAGE_LENGTH, MAX_TOPIC_NAME_LENGTH, - Message, Realm, Recipient, Stream, UserMessage, UserProfile, Attachment, - RealmAuditLog, RealmDomain, get_realm, UserPresence, Subscription, - get_stream, get_system_bot, get_user, Reaction, - flush_per_request_caches, ScheduledMessage, get_huddle_recipient, - bulk_get_huddle_user_ids, get_huddle_user_ids, - get_display_recipient, RealmFilter, -) - - +from zerver.lib.test_runner import slow from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone +from zerver.lib.topic import DB_TOPIC_NAME, LEGACY_PREV_TOPIC, TOPIC_LINKS, TOPIC_NAME +from zerver.lib.types import DisplayRecipientT, UserDisplayRecipient from zerver.lib.upload import create_attachment from zerver.lib.url_encoding import near_message_url +from zerver.models import ( + MAX_MESSAGE_LENGTH, + MAX_TOPIC_NAME_LENGTH, + Attachment, + Message, + Reaction, + Realm, + RealmAuditLog, + RealmDomain, + RealmFilter, + Recipient, + ScheduledMessage, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + bulk_get_huddle_user_ids, + flush_per_request_caches, + get_display_recipient, + get_huddle_recipient, + get_huddle_user_ids, + get_realm, + get_stream, + get_system_bot, + get_user, +) +from zerver.views.messages import InvalidMirrorInput, create_mirrored_message_users -from zerver.views.messages import create_mirrored_message_users, InvalidMirrorInput - -from analytics.lib.counts import COUNT_STATS -from analytics.models import RealmCount - -import datetime -from unittest import mock -from operator import itemgetter -import time -import ujson -from typing import Any, Dict, List, Set, Union, Tuple - -from collections import namedtuple class MiscMessageTest(ZulipTestCase): def test_get_last_message_id(self) -> None: diff --git a/zerver/tests/test_middleware.py b/zerver/tests/test_middleware.py index 1e5c0a440f..6506b73f04 100644 --- a/zerver/tests/test_middleware.py +++ b/zerver/tests/test_middleware.py @@ -1,13 +1,15 @@ import time from typing import List +from unittest.mock import patch from bs4 import BeautifulSoup -from unittest.mock import patch + from zerver.lib.realm_icon import get_realm_icon_url from zerver.lib.test_classes import ZulipTestCase from zerver.middleware import is_slow_query, write_log_line from zerver.models import get_realm + class SlowQueryTest(ZulipTestCase): SLOW_QUERY_TIME = 10 log_data = {'extra': '[transport=websocket]', diff --git a/zerver/tests/test_migrations.py b/zerver/tests/test_migrations.py index 7382f613a2..a83f190b9a 100644 --- a/zerver/tests/test_migrations.py +++ b/zerver/tests/test_migrations.py @@ -4,11 +4,10 @@ # You can also read # https://www.caktusgroup.com/blog/2016/02/02/writing-unit-tests-django-migrations/ # to get a tutorial on the framework that inspired this feature. +from django.db.migrations.state import StateApps from zerver.lib.test_classes import MigrationsTestCase from zerver.lib.test_helpers import use_db_models -from django.db.migrations.state import StateApps - from zerver.models import get_stream # Important note: These tests are very expensive, and details of diff --git a/zerver/tests/test_muting.py b/zerver/tests/test_muting.py index 5a64ee928e..91d72ff386 100644 --- a/zerver/tests/test_muting.py +++ b/zerver/tests/test_muting.py @@ -1,23 +1,19 @@ -from django.utils.timezone import now as timezone_now -from datetime import timedelta, datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict from unittest import mock -from zerver.lib.test_classes import ZulipTestCase +from django.utils.timezone import now as timezone_now + from zerver.lib.stream_topic import StreamTopicTarget - -from zerver.models import ( - get_stream, - UserProfile, - MutedTopic, -) - +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic_mutes import ( add_topic_mute, get_topic_mutes, remove_topic_mute, topic_is_muted, ) +from zerver.models import MutedTopic, UserProfile, get_stream + class MutedTopicsTests(ZulipTestCase): def test_user_ids_muting_topic(self) -> None: diff --git a/zerver/tests/test_narrow.py b/zerver/tests/test_narrow.py index a118e44491..0eaf11bae6 100644 --- a/zerver/tests/test_narrow.py +++ b/zerver/tests/test_narrow.py @@ -1,57 +1,47 @@ +import os +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union +from unittest import mock + +import ujson from django.db import connection from django.test import TestCase, override_settings -from sqlalchemy.sql import ( - and_, select, column, table, -) +from sqlalchemy.sql import and_, column, select, table from sqlalchemy.sql.elements import ClauseElement -from zerver.models import ( - Realm, Subscription, Recipient, Stream, - get_display_recipient, get_realm, get_stream, - UserMessage, Message, -) -from zerver.lib.actions import ( - do_set_realm_property, - do_deactivate_user, -) -from zerver.lib.message import ( - MessageDict, -) -from zerver.lib.narrow import ( - build_narrow_filter, - is_web_public_compatible, -) +from zerver.lib.actions import do_deactivate_user, do_set_realm_property +from zerver.lib.message import MessageDict +from zerver.lib.narrow import build_narrow_filter, is_web_public_compatible from zerver.lib.request import JsonableError from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.test_helpers import ( - POSTRequestMock, - get_user_messages, queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.topic import ( - MATCH_TOPIC, - TOPIC_NAME, -) -from zerver.lib.topic_mutes import ( - set_topic_mutes, -) -from zerver.lib.types import DisplayRecipientT -from zerver.views.messages import ( - exclude_muting_conditions, - get_messages_backend, ok_to_include_history, - NarrowBuilder, BadNarrowOperator, Query, - post_process_limited_query, - find_first_unread_anchor, - LARGER_THAN_MAX_MESSAGE_ID, -) from zerver.lib.streams import create_streams_if_needed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import POSTRequestMock, get_user_messages, queries_captured +from zerver.lib.topic import MATCH_TOPIC, TOPIC_NAME +from zerver.lib.topic_mutes import set_topic_mutes +from zerver.lib.types import DisplayRecipientT +from zerver.models import ( + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + get_display_recipient, + get_realm, + get_stream, +) +from zerver.views.messages import ( + LARGER_THAN_MAX_MESSAGE_ID, + BadNarrowOperator, + NarrowBuilder, + Query, + exclude_muting_conditions, + find_first_unread_anchor, + get_messages_backend, + ok_to_include_history, + post_process_limited_query, +) -from typing import Dict, Mapping, List, Sequence, Tuple, Union, Any, Optional -from unittest import mock -import os -import ujson def get_sqlalchemy_sql(query: ClauseElement) -> str: dialect = get_sqlalchemy_connection().dialect diff --git a/zerver/tests/test_new_users.py b/zerver/tests/test_new_users.py index ec1baccb00..9cc3778ab6 100644 --- a/zerver/tests/test_new_users.py +++ b/zerver/tests/test_new_users.py @@ -1,15 +1,17 @@ import datetime +from unittest import mock + from django.conf import settings from django.core import mail from django.test import override_settings -from zerver.lib.test_classes import ZulipTestCase -from zerver.signals import get_device_browser, get_device_os, JUST_CREATED_THRESHOLD -from zerver.lib.actions import notify_new_user, do_change_notification_settings -from zerver.models import Recipient, Stream, Realm +from zerver.lib.actions import do_change_notification_settings, notify_new_user from zerver.lib.initial_password import initial_password -from unittest import mock +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timezone import get_timezone +from zerver.models import Realm, Recipient, Stream +from zerver.signals import JUST_CREATED_THRESHOLD, get_device_browser, get_device_os + class SendLoginEmailTest(ZulipTestCase): """ diff --git a/zerver/tests/test_onboarding.py b/zerver/tests/test_onboarding.py index 4477257c20..ace78fcd29 100644 --- a/zerver/tests/test_onboarding.py +++ b/zerver/tests/test_onboarding.py @@ -1,8 +1,7 @@ -from zerver.models import Realm, UserProfile from zerver.lib.onboarding import create_if_missing_realm_internal_bots -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile + class TestRealmInternalBotCreation(ZulipTestCase): def test_create_if_missing_realm_internal_bots(self) -> None: diff --git a/zerver/tests/test_openapi.py b/zerver/tests/test_openapi.py index 7a7f8fff1a..03dc02bdd0 100644 --- a/zerver/tests/test_openapi.py +++ b/zerver/tests/test_openapi.py @@ -1,24 +1,43 @@ +import inspect import re import sys +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Union, +) from unittest import mock -import inspect -from typing import Dict, Any, Set, Union, List, Callable, Tuple, Optional, Iterable, Mapping, Sequence -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from django.http import HttpResponse -import zerver.openapi.openapi as openapi -from zerver.openapi.markdown_extension import generate_curl_example, \ - render_curl_example, parse_language_and_options -from zerver.lib.request import _REQ +from zerver.lib.request import _REQ, arguments_map from zerver.lib.test_classes import ZulipTestCase -from zerver.openapi.openapi import ( - get_openapi_fixture, get_openapi_parameters, - validate_against_openapi_schema, to_python_type, - SchemaError, openapi_spec, get_openapi_paths, - OpenAPISpec, OPENAPI_SPEC_PATH, +from zerver.openapi import openapi as openapi +from zerver.openapi.markdown_extension import ( + generate_curl_example, + parse_language_and_options, + render_curl_example, +) +from zerver.openapi.openapi import ( + OPENAPI_SPEC_PATH, + OpenAPISpec, + SchemaError, + get_openapi_fixture, + get_openapi_parameters, + get_openapi_paths, + openapi_spec, + to_python_type, + validate_against_openapi_schema, ) -from zerver.lib.request import arguments_map TEST_ENDPOINT = '/messages/{message_id}' TEST_METHOD = 'patch' @@ -525,7 +544,7 @@ do not match the types declared in the implementation of {}.\n""".format(functio in code. """ - import zproject.urls as urlconf + from zproject import urls as urlconf # We loop through all the API patterns, looking in particular # for those using the rest_dispatch decorator; we then parse diff --git a/zerver/tests/test_outgoing_webhook_interfaces.py b/zerver/tests/test_outgoing_webhook_interfaces.py index 5e141ec8b0..75b8ecdc87 100644 --- a/zerver/tests/test_outgoing_webhook_interfaces.py +++ b/zerver/tests/test_outgoing_webhook_interfaces.py @@ -1,25 +1,16 @@ -from typing import cast, Any, Dict - -from unittest import mock import json +from typing import Any, Dict, cast +from unittest import mock + import requests from zerver.lib.avatar import get_gravatar_url from zerver.lib.message import MessageDict -from zerver.lib.outgoing_webhook import ( - get_service_interface_class, - process_success_response, -) +from zerver.lib.outgoing_webhook import get_service_interface_class, process_success_response from zerver.lib.test_classes import ZulipTestCase from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.topic import TOPIC_NAME -from zerver.models import ( - get_realm, - get_stream, - get_user, - Message, - SLACK_INTERFACE, -) +from zerver.models import SLACK_INTERFACE, Message, get_realm, get_stream, get_user class TestGenericOutgoingWebhookService(ZulipTestCase): diff --git a/zerver/tests/test_outgoing_webhook_system.py b/zerver/tests/test_outgoing_webhook_system.py index aee46890ae..b0c30cf3f4 100644 --- a/zerver/tests/test_outgoing_webhook_system.py +++ b/zerver/tests/test_outgoing_webhook_system.py @@ -1,33 +1,29 @@ -import ujson import logging -from unittest import mock -import requests - from typing import Any, Optional +from unittest import mock -from zerver.lib.actions import ( - do_create_user, -) +import requests +import ujson +from version import ZULIP_VERSION +from zerver.lib.actions import do_create_user from zerver.lib.outgoing_webhook import ( - do_rest_call, GenericOutgoingWebhookService, SlackOutgoingWebhookService, + do_rest_call, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.topic import TOPIC_NAME from zerver.lib.users import add_service from zerver.models import ( - get_display_recipient, - get_realm, - get_user, Recipient, Service, UserProfile, + get_display_recipient, + get_realm, + get_user, ) -from version import ZULIP_VERSION class ResponseMock: def __init__(self, status_code: int, content: Optional[Any]=None) -> None: diff --git a/zerver/tests/test_populate_db.py b/zerver/tests/test_populate_db.py index 31c946c050..1111d73858 100644 --- a/zerver/tests/test_populate_db.py +++ b/zerver/tests/test_populate_db.py @@ -1,8 +1,9 @@ -from zilencer.management.commands.populate_db import choose_date_sent -from zerver.lib.test_classes import ZulipTestCase - from django.utils.timezone import timedelta as timezone_timedelta +from zerver.lib.test_classes import ZulipTestCase +from zilencer.management.commands.populate_db import choose_date_sent + + class TestChoosePubDate(ZulipTestCase): def test_choose_date_sent_large_tot_messages(self) -> None: """ diff --git a/zerver/tests/test_presence.py b/zerver/tests/test_presence.py index 9a16db32d6..52cb221ee9 100644 --- a/zerver/tests/test_presence.py +++ b/zerver/tests/test_presence.py @@ -1,33 +1,26 @@ +import datetime from datetime import timedelta -from django.utils.timezone import now as timezone_now +from typing import Any, Dict from unittest import mock -from typing import Any, Dict +from django.utils.timezone import now as timezone_now + from zerver.lib.actions import do_deactivate_user -from zerver.lib.presence import ( - get_status_dict_by_realm, -) +from zerver.lib.presence import get_status_dict_by_realm from zerver.lib.statistics import seconds_usage_between -from zerver.lib.test_helpers import ( - make_client, - queries_captured, - reset_emails_in_zulip_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import make_client, queries_captured, reset_emails_in_zulip_realm from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import ( Client, PushDeviceToken, UserActivity, UserActivityInterval, - UserProfile, UserPresence, + UserProfile, flush_per_request_caches, ) -import datetime class ActivityTest(ZulipTestCase): @mock.patch("stripe.Customer.list", return_value=[]) diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index b2097566fc..6255fbc1db 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -1,52 +1,35 @@ -from contextlib import contextmanager +import base64 import datetime import itertools -import requests +import os +import uuid +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Optional from unittest import mock from unittest.mock import call -from typing import Any, Dict, Iterator, List, Optional -import base64 -import os +import requests import ujson -import uuid - -from django.test import override_settings from django.conf import settings -from django.http import HttpResponse from django.db import transaction from django.db.models import F +from django.http import HttpResponse +from django.test import override_settings from django.utils.crypto import get_random_string +from django.utils.timezone import now from analytics.lib.counts import CountStat, LoggingCountStat from analytics.models import InstallationCount, RealmCount -from zerver.models import ( - PushDeviceToken, - Message, - UserMessage, - receives_offline_email_notifications, - receives_offline_push_notifications, - receives_online_notifications, - receives_stream_notifications, - get_client, - get_realm, - get_stream, - Recipient, - RealmAuditLog, - Stream, - Subscription, -) from zerver.lib.actions import ( do_delete_messages, do_mark_stream_messages_as_read, do_regenerate_api_key, ) -from zerver.lib.soft_deactivation import do_soft_deactivate_users from zerver.lib.push_notifications import ( + DeviceToken, absolute_avatar_url, b64_to_hex, datetime_to_timestamp, - DeviceToken, get_apns_client, get_display_recipient, get_message_payload_apns, @@ -63,16 +46,38 @@ from zerver.lib.push_notifications import ( send_notifications_to_bouncer, send_to_push_bouncer, ) -from zerver.lib.remote_server import send_analytics_to_remote_server, \ - build_analytics_data, PushNotificationBouncerException, PushNotificationBouncerRetryLaterError -from zerver.lib.request import JsonableError -from zerver.lib.test_classes import ( - TestCase, ZulipTestCase, +from zerver.lib.remote_server import ( + PushNotificationBouncerException, + PushNotificationBouncerRetryLaterError, + build_analytics_data, + send_analytics_to_remote_server, +) +from zerver.lib.request import JsonableError +from zerver.lib.soft_deactivation import do_soft_deactivate_users +from zerver.lib.test_classes import TestCase, ZulipTestCase +from zerver.models import ( + Message, + PushDeviceToken, + RealmAuditLog, + Recipient, + Stream, + Subscription, + UserMessage, + get_client, + get_realm, + get_stream, + receives_offline_email_notifications, + receives_offline_push_notifications, + receives_online_notifications, + receives_stream_notifications, +) +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, ) - -from zilencer.models import RemoteZulipServer, RemotePushDeviceToken, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog -from django.utils.timezone import now ZERVER_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -1510,6 +1515,7 @@ class TestSendToPushBouncer(ZulipTestCase): def test_400_error_invalid_server_key(self) -> None: from zerver.decorator import InvalidZulipServerError + # This is the exception our decorator uses for an invalid Zulip server error_obj = InvalidZulipServerError("testRole") with mock.patch('requests.request', diff --git a/zerver/tests/test_queue.py b/zerver/tests/test_queue.py index 48f36d0f7f..d08b43e172 100644 --- a/zerver/tests/test_queue.py +++ b/zerver/tests/test_queue.py @@ -1,13 +1,13 @@ -from unittest import mock from typing import Any, Dict +from unittest import mock from django.test import override_settings -from pika.exceptions import ConnectionClosed, AMQPConnectionError +from pika.exceptions import AMQPConnectionError, ConnectionClosed -from zerver.lib.queue import TornadoQueueClient, queue_json_publish, \ - get_queue_client +from zerver.lib.queue import TornadoQueueClient, get_queue_client, queue_json_publish from zerver.lib.test_classes import ZulipTestCase + class TestTornadoQueueClient(ZulipTestCase): @mock.patch('zerver.lib.queue.logging.getLogger', autospec=True) @mock.patch('zerver.lib.queue.ExceptionFreeTornadoConnection', autospec=True) diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index 2bde78a4c6..4bfff4fdf5 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -1,12 +1,12 @@ import os -import time -import ujson import smtplib +import time +from typing import Any, Callable, Dict, List, Mapping, Tuple +from unittest.mock import MagicMock, patch +import ujson from django.conf import settings from django.test import override_settings -from unittest.mock import patch, MagicMock -from typing import Any, Callable, Dict, List, Mapping, Tuple from zerver.lib.email_mirror import RateLimitedRealmMirror from zerver.lib.email_mirror_helpers import encode_email_address @@ -14,18 +14,17 @@ from zerver.lib.queue import MAX_REQUEST_RETRIES from zerver.lib.rate_limiter import RateLimiterLockingException from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError from zerver.lib.send_email import FromAddress -from zerver.lib.test_helpers import simulated_queue_client from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_client, UserActivity, PreregistrationUser, \ - get_stream, get_realm +from zerver.lib.test_helpers import simulated_queue_client +from zerver.models import PreregistrationUser, UserActivity, get_client, get_realm, get_stream from zerver.tornado.event_queue import build_offline_notification from zerver.worker import queue_processors from zerver.worker.queue_processors import ( - get_active_worker_queues, - QueueProcessingWorker, EmailSendingWorker, LoopQueueProcessingWorker, MissedMessageWorker, + QueueProcessingWorker, + get_active_worker_queues, ) Event = Dict[str, Any] diff --git a/zerver/tests/test_rate_limiter.py b/zerver/tests/test_rate_limiter.py index 7afa2379f4..898af5fb37 100644 --- a/zerver/tests/test_rate_limiter.py +++ b/zerver/tests/test_rate_limiter.py @@ -1,21 +1,19 @@ +import time +from typing import Dict, List, Tuple, Type +from unittest import mock + from zerver.lib.rate_limiter import ( - add_ratelimit_rule, - remove_ratelimit_rule, RateLimitedObject, RateLimitedUser, RateLimiterBackend, RedisRateLimiterBackend, TornadoInMemoryRateLimiterBackend, + add_ratelimit_rule, + remove_ratelimit_rule, ) - from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import generate_random_token -from typing import Dict, List, Tuple, Type - -from unittest import mock -import time - RANDOM_KEY_PREFIX = generate_random_token(32) class RateLimitedTestObject(RateLimitedObject): diff --git a/zerver/tests/test_reactions.py b/zerver/tests/test_reactions.py index 9e1ad910ee..ff349a3fcc 100644 --- a/zerver/tests/test_reactions.py +++ b/zerver/tests/test_reactions.py @@ -1,15 +1,17 @@ -import ujson -from django.http import HttpResponse from typing import Any, Dict, List, Mapping from unittest import mock -from zerver.lib.cache import to_dict_cache_key_id, cache_get +import ujson +from django.http import HttpResponse + +from zerver.lib.cache import cache_get, to_dict_cache_key_id from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.message import extract_message_dict from zerver.lib.request import JsonableError -from zerver.lib.test_helpers import tornado_redirected_to_list from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, Message, Reaction, RealmEmoji, UserMessage +from zerver.lib.test_helpers import tornado_redirected_to_list +from zerver.models import Message, Reaction, RealmEmoji, UserMessage, get_realm + class ReactionEmojiTest(ZulipTestCase): def test_missing_emoji(self) -> None: diff --git a/zerver/tests/test_realm.py b/zerver/tests/test_realm.py index 89a514cceb..e571281bde 100644 --- a/zerver/tests/test_realm.py +++ b/zerver/tests/test_realm.py @@ -1,36 +1,43 @@ import datetime -import ujson import re -from unittest import mock from email.utils import parseaddr - -from django.conf import settings from typing import Any, Dict, List, Mapping +from unittest import mock +import ujson +from django.conf import settings + +from confirmation.models import Confirmation, create_confirmation_link from zerver.lib.actions import ( + do_change_plan_type, do_change_realm_subdomain, - do_set_realm_property, + do_create_realm, do_deactivate_realm, do_deactivate_stream, - do_create_realm, do_scrub_realm, - do_change_plan_type, do_send_realm_reactivation_email, + do_set_realm_property, ) - -from confirmation.models import create_confirmation_link, Confirmation from zerver.lib.realm_description import get_realm_rendered_description, get_realm_text_description from zerver.lib.send_email import send_future_email from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - reset_emails_in_zulip_realm, - tornado_redirected_to_list, -) +from zerver.lib.test_helpers import reset_emails_in_zulip_realm, tornado_redirected_to_list from zerver.lib.test_runner import slow -from zerver.models import get_realm, Realm, UserProfile, ScheduledEmail, get_stream, \ - CustomProfileField, Message, UserMessage, Attachment, get_user_profile_by_email, \ - get_user_profile_by_id +from zerver.models import ( + Attachment, + CustomProfileField, + Message, + Realm, + ScheduledEmail, + UserMessage, + UserProfile, + get_realm, + get_stream, + get_user_profile_by_email, + get_user_profile_by_id, +) + class RealmTest(ZulipTestCase): def assert_user_profile_cache_gets_new_name(self, user_profile: UserProfile, diff --git a/zerver/tests/test_realm_domains.py b/zerver/tests/test_realm_domains.py index 872fd78f9d..65a64c8d7e 100644 --- a/zerver/tests/test_realm_domains.py +++ b/zerver/tests/test_realm_domains.py @@ -1,16 +1,18 @@ +import ujson from django.core.exceptions import ValidationError from django.db.utils import IntegrityError -from zerver.lib.actions import do_change_user_role, \ - do_change_realm_domain, do_create_realm, \ - do_remove_realm_domain, do_set_realm_property -from zerver.lib.email_validation import email_allowed_for_realm +from zerver.lib.actions import ( + do_change_realm_domain, + do_change_user_role, + do_create_realm, + do_remove_realm_domain, + do_set_realm_property, +) from zerver.lib.domains import validate_domain +from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.test_classes import ZulipTestCase -from zerver.models import get_realm, UserProfile, \ - RealmDomain, DomainNotAllowedForRealmError - -import ujson +from zerver.models import DomainNotAllowedForRealmError, RealmDomain, UserProfile, get_realm class RealmDomainTest(ZulipTestCase): diff --git a/zerver/tests/test_realm_emoji.py b/zerver/tests/test_realm_emoji.py index 0a672339c7..4874fb1fa9 100644 --- a/zerver/tests/test_realm_emoji.py +++ b/zerver/tests/test_realm_emoji.py @@ -1,11 +1,11 @@ from unittest import mock -from zerver.lib.actions import do_create_realm, do_create_user, \ - check_add_realm_emoji +from zerver.lib.actions import check_add_realm_emoji, do_create_realm, do_create_user from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.models import Realm, RealmEmoji, UserProfile, get_realm + class RealmEmojiTest(ZulipTestCase): def create_test_emoji(self, name: str, author: UserProfile) -> RealmEmoji: diff --git a/zerver/tests/test_realm_export.py b/zerver/tests/test_realm_export.py index 0c0a040699..e9af3781dd 100644 --- a/zerver/tests/test_realm_export.py +++ b/zerver/tests/test_realm_export.py @@ -1,21 +1,23 @@ +import os from unittest.mock import patch -from analytics.models import RealmCount - -from django.utils.timezone import now as timezone_now +import botocore.exceptions +import ujson from django.conf import settings +from django.utils.timezone import now as timezone_now -from zerver.lib.test_classes import ZulipTestCase +from analytics.models import RealmCount from zerver.lib.exceptions import JsonableError -from zerver.lib.test_helpers import use_s3_backend, create_s3_buckets, \ - create_dummy_file, stdout_suppressed - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + create_dummy_file, + create_s3_buckets, + stdout_suppressed, + use_s3_backend, +) from zerver.models import RealmAuditLog from zerver.views.realm_export import export_realm -import os -import ujson -import botocore.exceptions class RealmExportTest(ZulipTestCase): """ diff --git a/zerver/tests/test_realm_filters.py b/zerver/tests/test_realm_filters.py index f94c2b1881..7b93ece2c5 100644 --- a/zerver/tests/test_realm_filters.py +++ b/zerver/tests/test_realm_filters.py @@ -4,6 +4,7 @@ from zerver.lib.actions import do_add_realm_filter from zerver.lib.test_classes import ZulipTestCase from zerver.models import RealmFilter, get_realm + class RealmFilterTest(ZulipTestCase): def test_list(self) -> None: diff --git a/zerver/tests/test_redis_utils.py b/zerver/tests/test_redis_utils.py index db15268f61..c69a4de40c 100644 --- a/zerver/tests/test_redis_utils.py +++ b/zerver/tests/test_redis_utils.py @@ -1,8 +1,15 @@ from unittest import mock +from zerver.lib.redis_utils import ( + MAX_KEY_LENGTH, + ZulipRedisKeyOfWrongFormatError, + ZulipRedisKeyTooLongError, + get_dict_from_redis, + get_redis_client, + put_dict_in_redis, +) from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis, \ - ZulipRedisKeyTooLongError, ZulipRedisKeyOfWrongFormatError, MAX_KEY_LENGTH + class RedisUtilsTest(ZulipTestCase): key_format = "test_redis_utils_{token}" diff --git a/zerver/tests/test_report.py b/zerver/tests/test_report.py index 714e70adfe..46d3b4f680 100644 --- a/zerver/tests/test_report.py +++ b/zerver/tests/test_report.py @@ -1,13 +1,12 @@ from typing import Any, Callable, Dict, Iterable, List, Tuple +from unittest import mock +import ujson from django.test import override_settings -from zerver.lib.test_classes import ( - ZulipTestCase, -) + +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.utils import statsd -from unittest import mock -import ujson def fix_params(raw_params: Dict[str, Any]) -> Dict[str, str]: # A few of our few legacy endpoints need their diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 056909aec8..192ab15455 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -5,25 +5,39 @@ from unittest import mock from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.actions import internal_send_private_message, do_add_submessage, do_delete_messages +from zerver.lib.actions import do_add_submessage, do_delete_messages, internal_send_private_message +from zerver.lib.retention import ( + archive_messages, + clean_archived_data, + get_realms_and_streams_for_archiving, + move_messages_to_archive, + restore_all_data_from_archive, +) from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import queries_captured from zerver.lib.upload import create_attachment -from zerver.models import (Message, Realm, Stream, ArchivedUserMessage, SubMessage, - ArchivedMessage, Attachment, ArchivedAttachment, UserMessage, - Reaction, ArchivedReaction, ArchivedSubMessage, ArchiveTransaction, - get_realm, get_user_profile_by_email, get_stream, get_system_bot) -from zerver.lib.retention import ( - archive_messages, - move_messages_to_archive, - restore_all_data_from_archive, - clean_archived_data, - get_realms_and_streams_for_archiving, +from zerver.models import ( + ArchivedAttachment, + ArchivedMessage, + ArchivedReaction, + ArchivedSubMessage, + ArchivedUserMessage, + ArchiveTransaction, + Attachment, + Message, + Reaction, + Realm, + Stream, + SubMessage, + UserMessage, + get_realm, + get_stream, + get_system_bot, + get_user_profile_by_email, ) -from zerver.tornado.event_queue import send_event - # Class with helper functions useful for testing archiving of reactions: from zerver.tests.test_reactions import EmojiReactionBase +from zerver.tornado.event_queue import send_event ZULIP_REALM_DAYS = 30 MIT_REALM_DAYS = 100 diff --git a/zerver/tests/test_service_bot_system.py b/zerver/tests/test_service_bot_system.py index 8867ef75d2..6978640246 100644 --- a/zerver/tests/test_service_bot_system.py +++ b/zerver/tests/test_service_bot_system.py @@ -1,25 +1,16 @@ +from typing import Any, Callable, Mapping, Union from unittest import mock -from typing import Any, Union, Mapping, Callable +import ujson from django.conf import settings from django.test import override_settings -from zerver.lib.actions import ( - do_create_user, - get_service_bot_events, -) -from zerver.lib.bot_lib import StateHandler, EmbeddedBotHandler, \ - EmbeddedBotEmptyRecipientsList +from zerver.lib.actions import do_create_user, get_service_bot_events +from zerver.lib.bot_config import ConfigError, load_bot_config_template, set_bot_config +from zerver.lib.bot_lib import EmbeddedBotEmptyRecipientsList, EmbeddedBotHandler, StateHandler from zerver.lib.bot_storage import StateError -from zerver.lib.bot_config import set_bot_config, ConfigError, load_bot_config_template from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - get_realm, - UserProfile, - Recipient, -) - -import ujson +from zerver.models import Recipient, UserProfile, get_realm BOT_TYPE_TO_QUEUE_NAME = { UserProfile.OUTGOING_WEBHOOK_BOT: 'outgoing_webhooks', diff --git a/zerver/tests/test_sessions.py b/zerver/tests/test_sessions.py index 9b4daf3294..5b096201a4 100644 --- a/zerver/tests/test_sessions.py +++ b/zerver/tests/test_sessions.py @@ -1,25 +1,22 @@ from datetime import timedelta -from django.utils.timezone import now as timezone_now from typing import Any, Callable +from unittest import mock + +from django.utils.timezone import now as timezone_now from zerver.lib.sessions import ( - user_sessions, + delete_all_deactivated_user_sessions, + delete_all_user_sessions, + delete_realm_user_sessions, delete_session, delete_user_sessions, - delete_realm_user_sessions, - delete_all_user_sessions, - delete_all_deactivated_user_sessions, get_expirable_session_var, set_expirable_session_var, + user_sessions, ) - -from zerver.models import ( - get_realm, Realm, UserProfile, -) - from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Realm, UserProfile, get_realm -from unittest import mock class TestSessions(ZulipTestCase): diff --git a/zerver/tests/test_settings.py b/zerver/tests/test_settings.py index 4b70aa01d0..246ae6f687 100644 --- a/zerver/tests/test_settings.py +++ b/zerver/tests/test_settings.py @@ -1,20 +1,18 @@ -from unittest import mock import time -import ujson +from typing import Any, Dict +from unittest import mock +import ujson from django.http import HttpResponse from django.test import override_settings -from typing import Any, Dict from zerver.lib.initial_password import initial_password +from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import get_test_image_file from zerver.lib.users import get_all_api_keys -from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.models import ( - UserProfile, - get_user_profile_by_api_key, -) +from zerver.models import UserProfile, get_user_profile_by_api_key + class ChangeSettingsTest(ZulipTestCase): diff --git a/zerver/tests/test_signup.py b/zerver/tests/test_signup.py index 242e007862..9bc375b8ac 100644 --- a/zerver/tests/test_signup.py +++ b/zerver/tests/test_signup.py @@ -1,86 +1,102 @@ import datetime -from email.utils import parseaddr - -from django.conf import settings -from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN -from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponse -from django.test import TestCase, override_settings -from django.utils.timezone import now as timezone_now -from django.core.exceptions import ValidationError -from django.urls import reverse - -from unittest.mock import patch, MagicMock -from zerver.lib.test_helpers import ( - avatar_disk_path, - get_test_image_file, - reset_emails_in_zulip_realm, -) - -from confirmation.models import Confirmation, create_confirmation_link, MultiuseInvite, \ - generate_key, confirmation_url, get_object_from_key, ConfirmationKeyException, \ - one_click_unsubscribe_link -from confirmation import settings as confirmation_settings - -from zerver.forms import HomepageForm, check_subdomain_available -from zerver.decorator import do_two_factor_login -from zerver.views.auth import \ - redirect_and_log_into_subdomain, start_two_factor_auth -from zerver.views.invite import get_invitee_emails_set -from zerver.views.development.registration import confirmation_key - -from zerver.models import ( - get_realm, get_user, CustomProfileField, - CustomProfileFieldValue, DefaultStream, PreregistrationUser, - Realm, Recipient, Message, ScheduledEmail, UserProfile, UserMessage, - Stream, Subscription, flush_per_request_caches, get_system_bot, - get_user_by_delivery_email, -) -from zerver.lib.actions import ( - do_change_user_role, - get_stream, - do_create_default_stream_group, - do_add_default_stream, - do_create_realm, - get_default_streams_for_realm, - do_invite_users, do_create_user) -from zerver.lib.send_email import send_future_email, FromAddress, \ - deliver_email -from zerver.lib.initial_password import initial_password -from zerver.lib.actions import ( - do_get_user_invites, - do_change_full_name, - do_deactivate_realm, - do_deactivate_user, - do_set_realm_property, - add_new_user_history, -) -from zerver.lib.mobile_auth_otp import xor_hex_strings, ascii_to_hex, \ - otp_encrypt_api_key, is_valid_otp, hex_to_ascii, otp_decrypt_api_key -from zerver.lib.email_notifications import enqueue_welcome_emails, \ - followup_day2_email_delay -from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule -from zerver.lib.subdomains import is_root_domain_available -from zerver.lib.stream_subscription import get_stream_subscriptions_for_user -from zerver.lib.streams import create_stream_if_needed -from zerver.lib.test_helpers import find_key_by_email, queries_captured, \ - HostRequestMock, load_subdomain_token -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.name_restrictions import is_disposable_domain -from zerver.context_processors import common_context - -from zproject.backends import ExternalAuthResult, ExternalAuthDataDict - import re import smtplib import time -import ujson - -from typing import Any, List, Optional - import urllib +from email.utils import parseaddr +from typing import Any, List, Optional +from unittest.mock import MagicMock, patch + +import ujson +from django.conf import settings +from django.contrib.auth.views import INTERNAL_RESET_URL_TOKEN +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.http import HttpResponse +from django.test import TestCase, override_settings +from django.urls import reverse +from django.utils.timezone import now as timezone_now + +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + MultiuseInvite, + confirmation_url, + create_confirmation_link, + generate_key, + get_object_from_key, + one_click_unsubscribe_link, +) +from zerver.context_processors import common_context +from zerver.decorator import do_two_factor_login +from zerver.forms import HomepageForm, check_subdomain_available +from zerver.lib.actions import ( + add_new_user_history, + do_add_default_stream, + do_change_full_name, + do_change_user_role, + do_create_default_stream_group, + do_create_realm, + do_create_user, + do_deactivate_realm, + do_deactivate_user, + do_get_user_invites, + do_invite_users, + do_set_realm_property, + get_default_streams_for_realm, + get_stream, +) +from zerver.lib.email_notifications import enqueue_welcome_emails, followup_day2_email_delay +from zerver.lib.initial_password import initial_password +from zerver.lib.mobile_auth_otp import ( + ascii_to_hex, + hex_to_ascii, + is_valid_otp, + otp_decrypt_api_key, + otp_encrypt_api_key, + xor_hex_strings, +) +from zerver.lib.name_restrictions import is_disposable_domain +from zerver.lib.rate_limiter import add_ratelimit_rule, remove_ratelimit_rule +from zerver.lib.send_email import FromAddress, deliver_email, send_future_email +from zerver.lib.stream_subscription import get_stream_subscriptions_for_user +from zerver.lib.streams import create_stream_if_needed +from zerver.lib.subdomains import is_root_domain_available +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + HostRequestMock, + avatar_disk_path, + find_key_by_email, + get_test_image_file, + load_subdomain_token, + queries_captured, + reset_emails_in_zulip_realm, +) +from zerver.models import ( + CustomProfileField, + CustomProfileFieldValue, + DefaultStream, + Message, + PreregistrationUser, + Realm, + Recipient, + ScheduledEmail, + Stream, + Subscription, + UserMessage, + UserProfile, + flush_per_request_caches, + get_realm, + get_system_bot, + get_user, + get_user_by_delivery_email, +) +from zerver.views.auth import redirect_and_log_into_subdomain, start_two_factor_auth +from zerver.views.development.registration import confirmation_key +from zerver.views.invite import get_invitee_emails_set +from zproject.backends import ExternalAuthDataDict, ExternalAuthResult + class RedirectAndLogIntoSubdomainTestCase(ZulipTestCase): def test_data(self) -> None: @@ -3968,6 +3984,7 @@ class TestFindMyTeam(ZulipTestCase): self.assertIn(self.example_email("iago"), content) self.assertIn(self.example_email("cordelia"), content) from django.core.mail import outbox + # 3 = 1 + 2 -- Cordelia gets an email each for the "zulip" and "lear" realms. self.assertEqual(len(outbox), 3) diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 3d9b5df545..3e0ec99122 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -1,64 +1,49 @@ +import logging +import os +import shutil +from typing import Any, Dict, Iterator, List, Set, Tuple +from unittest import mock +from unittest.mock import ANY, call + +import ujson from django.conf import settings from django.utils.timezone import now as timezone_now +from zerver.data_import.import_util import ( + build_defaultstream, + build_recipient, + build_subscription, + build_usermessages, + build_zerver_realm, +) +from zerver.data_import.sequencer import NEXT_ID from zerver.data_import.slack import ( - get_slack_api_data, - get_owner, - get_admin, - get_guest, - get_user_timezone, - fetch_shared_channel_users, - users_to_zerver_userprofile, - get_subscription, - channels_to_zerver_stream, - slack_workspace_to_realm, - get_message_sending_user, - channel_message_to_zerver_message, - convert_slack_workspace_messages, - do_convert_data, - process_message_files, AddedChannelsT, AddedMPIMsT, DMMembersT, ZerverFieldsT, + channel_message_to_zerver_message, + channels_to_zerver_stream, + convert_slack_workspace_messages, + do_convert_data, + fetch_shared_channel_users, + get_admin, + get_guest, + get_message_sending_user, + get_owner, + get_slack_api_data, + get_subscription, + get_user_timezone, + process_message_files, + slack_workspace_to_realm, + users_to_zerver_userprofile, ) -from zerver.data_import.import_util import ( - build_zerver_realm, - build_subscription, - build_recipient, - build_usermessages, - build_defaultstream, -) -from zerver.data_import.sequencer import ( - NEXT_ID, -) -from zerver.lib.import_realm import ( - do_import_realm, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - get_test_image_file, -) -from zerver.lib.topic import ( - EXPORT_TOPIC_NAME, -) -from zerver.models import ( - Realm, - get_realm, - RealmAuditLog, - Recipient, - UserProfile, -) +from zerver.lib.import_realm import do_import_realm +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_test_image_file +from zerver.lib.topic import EXPORT_TOPIC_NAME +from zerver.models import Realm, RealmAuditLog, Recipient, UserProfile, get_realm -import ujson -import logging -import shutil -import os -from unittest import mock -from unittest.mock import ANY, call -from typing import Any, Dict, List, Set, Tuple, Iterator def remove_folder(path: str) -> None: if os.path.exists(path): diff --git a/zerver/tests/test_slack_message_conversion.py b/zerver/tests/test_slack_message_conversion.py index 11c6154590..2fc2063e0a 100644 --- a/zerver/tests/test_slack_message_conversion.py +++ b/zerver/tests/test_slack_message_conversion.py @@ -1,16 +1,16 @@ +import os +from typing import Any, Dict, List, Tuple + +import ujson + from zerver.data_import.slack_message_conversion import ( convert_to_zulip_markdown, get_user_full_name, ) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_runner import slow from zerver.lib import mdiff -import ujson +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_runner import slow -import os -from typing import Any, Dict, List, Tuple class SlackMessageConversion(ZulipTestCase): def assertEqual(self, first: Any, second: Any, msg: str="") -> None: diff --git a/zerver/tests/test_soft_deactivation.py b/zerver/tests/test_soft_deactivation.py index c7eda6dba8..45fb69fb79 100644 --- a/zerver/tests/test_soft_deactivation.py +++ b/zerver/tests/test_soft_deactivation.py @@ -3,18 +3,17 @@ from unittest import mock from django.utils.timezone import now as timezone_now from zerver.lib.soft_deactivation import ( + do_auto_soft_deactivate_users, + do_catch_up_soft_deactivated_users, + do_soft_activate_users, do_soft_deactivate_user, do_soft_deactivate_users, - get_users_for_soft_deactivation, - do_soft_activate_users, get_soft_deactivated_users_for_catch_up, - do_catch_up_soft_deactivated_users, - do_auto_soft_deactivate_users, + get_users_for_soft_deactivation, ) from zerver.lib.test_classes import ZulipTestCase -from zerver.models import ( - Client, UserProfile, UserActivity, get_realm, UserMessage, -) +from zerver.models import Client, UserActivity, UserMessage, UserProfile, get_realm + class UserSoftDeactivationTests(ZulipTestCase): diff --git a/zerver/tests/test_subdomains.py b/zerver/tests/test_subdomains.py index 2b6a37f875..25ad1edd90 100644 --- a/zerver/tests/test_subdomains.py +++ b/zerver/tests/test_subdomains.py @@ -1,11 +1,12 @@ -from unittest import mock from typing import Any, Dict, List +from unittest import mock from django.test import TestCase from zerver.lib.subdomains import get_subdomain from zerver.models import Realm + class SubdomainsTest(TestCase): def test_get_subdomain(self) -> None: diff --git a/zerver/tests/test_submessage.py b/zerver/tests/test_submessage.py index e3785c599c..e58620005b 100644 --- a/zerver/tests/test_submessage.py +++ b/zerver/tests/test_submessage.py @@ -1,17 +1,10 @@ +from typing import Any, Dict, List from unittest import mock +from zerver.lib.message import MessageDict from zerver.lib.test_classes import ZulipTestCase +from zerver.models import Message, SubMessage -from zerver.lib.message import ( - MessageDict, -) - -from zerver.models import ( - Message, - SubMessage, -) - -from typing import Any, Dict, List class TestBasics(ZulipTestCase): def test_get_raw_db_rows(self) -> None: diff --git a/zerver/tests/test_subs.py b/zerver/tests/test_subs.py index 6257055dcb..ffcfda63fe 100644 --- a/zerver/tests/test_subs.py +++ b/zerver/tests/test_subs.py @@ -1,87 +1,89 @@ +import random +from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Set, Union +from unittest import mock +import ujson from django.conf import settings from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now +from zerver.decorator import JsonableError from zerver.lib import cache - -from zerver.lib.test_helpers import ( - get_subscription, queries_captured, tornado_redirected_to_list, - reset_emails_in_zulip_realm, +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_get_subscriber_user_ids, + bulk_remove_subscriptions, + can_access_stream_user_ids, + create_stream_if_needed, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_post_policy, + do_change_user_role, + do_create_default_stream_group, + do_create_realm, + do_deactivate_stream, + do_deactivate_user, + do_get_streams, + do_remove_default_stream, + do_remove_default_stream_group, + do_remove_streams_from_default_stream_group, + do_set_realm_property, + ensure_stream, + gather_subscriptions, + gather_subscriptions_helper, + get_average_weekly_stream_traffic, + get_default_streams_for_realm, + get_stream, + lookup_default_stream_groups, + round_to_2_significant_digits, + validate_user_access_to_subscribers_helper, ) - -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.decorator import ( - JsonableError, -) - -from zerver.lib.response import ( - json_error, - json_success, -) - -from zerver.lib.streams import ( - access_stream_by_id, access_stream_by_name, filter_stream_authorization, - list_to_streams, create_streams_if_needed, -) - +from zerver.lib.message import aggregate_unread_data, get_raw_unread_data +from zerver.lib.response import json_error, json_success +from zerver.lib.stream_recipient import StreamRecipientMap from zerver.lib.stream_subscription import ( get_active_subscriptions_for_stream_id, num_subscribers_for_stream_id, ) - -from zerver.lib.test_runner import ( - slow, +from zerver.lib.streams import ( + access_stream_by_id, + access_stream_by_name, + create_streams_if_needed, + filter_stream_authorization, + list_to_streams, ) - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import ( + get_subscription, + queries_captured, + reset_emails_in_zulip_realm, + tornado_redirected_to_list, +) +from zerver.lib.test_runner import slow from zerver.models import ( - Realm, Recipient, Stream, Subscription, - DefaultStream, UserProfile, active_non_guest_user_ids, - get_default_stream_groups, flush_per_request_caches, DefaultStreamGroup, - get_client, get_realm, get_user, get_user_profile_by_id_in_realm, Message, UserMessage, + DefaultStream, + DefaultStreamGroup, + Message, + Realm, + Recipient, + Stream, + Subscription, + UserMessage, + UserProfile, + active_non_guest_user_ids, + flush_per_request_caches, + get_client, + get_default_stream_groups, + get_realm, + get_user, + get_user_profile_by_id_in_realm, ) +from zerver.views.streams import compose_views -from zerver.lib.actions import ( - do_add_default_stream, do_change_user_role, do_set_realm_property, - do_create_realm, do_remove_default_stream, bulk_get_subscriber_user_ids, - gather_subscriptions_helper, bulk_add_subscriptions, bulk_remove_subscriptions, - gather_subscriptions, get_default_streams_for_realm, get_stream, - do_get_streams, - create_stream_if_needed, - ensure_stream, - do_deactivate_stream, - do_deactivate_user, - do_create_default_stream_group, - do_add_streams_to_default_stream_group, do_remove_streams_from_default_stream_group, - do_remove_default_stream_group, - do_change_default_stream_group_description, - do_change_default_stream_group_name, - lookup_default_stream_groups, - can_access_stream_user_ids, - validate_user_access_to_subscribers_helper, - get_average_weekly_stream_traffic, round_to_2_significant_digits, - do_change_stream_post_policy, -) - -from zerver.views.streams import ( - compose_views, -) - -from zerver.lib.message import ( - aggregate_unread_data, - get_raw_unread_data, -) -from zerver.lib.stream_recipient import StreamRecipientMap - -from datetime import timedelta -from unittest import mock -import random -import ujson class TestMiscStuff(ZulipTestCase): def test_empty_results(self) -> None: diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index a35d282ab4..b5cdc86ee8 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -1,9 +1,7 @@ from django.template.loader import get_template from zerver.lib.exceptions import InvalidMarkdownIncludeStatement -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase class TemplateTestCase(ZulipTestCase): diff --git a/zerver/tests/test_thumbnail.py b/zerver/tests/test_thumbnail.py index 6653093bea..d707a28cf4 100644 --- a/zerver/tests/test_thumbnail.py +++ b/zerver/tests/test_thumbnail.py @@ -1,19 +1,20 @@ +import base64 +import urllib +from io import StringIO + +import ujson from django.conf import settings from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( - use_s3_backend, create_s3_buckets, - override_settings, get_test_image_file, + override_settings, + use_s3_backend, ) from zerver.lib.upload import upload_backend, upload_emoji_image from zerver.lib.users import get_api_key -from io import StringIO -import ujson -import urllib -import base64 class ThumbnailTest(ZulipTestCase): diff --git a/zerver/tests/test_timestamp.py b/zerver/tests/test_timestamp.py index cd2aa739c5..5460d510b5 100644 --- a/zerver/tests/test_timestamp.py +++ b/zerver/tests/test_timestamp.py @@ -1,11 +1,19 @@ -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.timestamp import floor_to_hour, floor_to_day, ceiling_to_hour, \ - timestamp_to_datetime, datetime_to_timestamp, \ - TimezoneNotUTCException, convert_to_UTC - from datetime import timedelta, timezone + from dateutil import parser +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.timestamp import ( + TimezoneNotUTCException, + ceiling_to_hour, + convert_to_UTC, + datetime_to_timestamp, + floor_to_day, + floor_to_hour, + timestamp_to_datetime, +) + + class TestTimestamp(ZulipTestCase): def test_datetime_and_timestamp_conversions(self) -> None: timestamp = 1483228800 diff --git a/zerver/tests/test_tornado.py b/zerver/tests/test_tornado.py index 9b377dddf6..bdb519e4b2 100644 --- a/zerver/tests/test_tornado.py +++ b/zerver/tests/test_tornado.py @@ -1,22 +1,20 @@ +import urllib.parse +from typing import Any, Dict, List, Optional, cast + import ujson from django.conf import settings -from django.db import close_old_connections from django.core import signals +from django.db import close_old_connections from django.test import override_settings from tornado.httpclient import HTTPResponse - -from zerver.lib.test_classes import ZulipTestCase - from tornado.testing import AsyncHTTPTestCase from tornado.web import Application -from zerver.tornado.application import create_tornado_application +from zerver.lib.test_classes import ZulipTestCase from zerver.tornado import event_queue +from zerver.tornado.application import create_tornado_application from zerver.tornado.event_queue import process_event -import urllib.parse - -from typing import Any, Dict, Optional, List, cast class TornadoWebTestCase(AsyncHTTPTestCase, ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_transfer.py b/zerver/tests/test_transfer.py index ec642b2b63..f1274e74d0 100644 --- a/zerver/tests/test_transfer.py +++ b/zerver/tests/test_transfer.py @@ -1,18 +1,22 @@ -from django.conf import settings - -from moto import mock_s3 -from unittest.mock import Mock, patch import logging +from unittest.mock import Mock, patch + +from django.conf import settings +from moto import mock_s3 -from zerver.lib.upload import upload_message_file, resize_emoji -from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import create_s3_buckets, get_test_image_file, \ - avatar_disk_path -from zerver.lib.transfer import transfer_avatars_to_s3, transfer_emoji_to_s3, \ - transfer_message_files_to_s3, transfer_uploads_to_s3 -from zerver.models import Attachment, RealmEmoji from zerver.lib.actions import check_add_realm_emoji from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import avatar_disk_path, create_s3_buckets, get_test_image_file +from zerver.lib.transfer import ( + transfer_avatars_to_s3, + transfer_emoji_to_s3, + transfer_message_files_to_s3, + transfer_uploads_to_s3, +) +from zerver.lib.upload import resize_emoji, upload_message_file +from zerver.models import Attachment, RealmEmoji + class TransferUploadsToS3Test(ZulipTestCase): logger = logging.getLogger() diff --git a/zerver/tests/test_tutorial.py b/zerver/tests/test_tutorial.py index 1ac2f6e06f..7332a25ee5 100644 --- a/zerver/tests/test_tutorial.py +++ b/zerver/tests/test_tutorial.py @@ -1,11 +1,11 @@ +import ujson from django.conf import settings from zerver.lib.actions import internal_send_private_message from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import message_stream_count, most_recent_message -from zerver.models import get_system_bot, UserProfile +from zerver.models import UserProfile, get_system_bot -import ujson class TutorialTests(ZulipTestCase): def setUp(self) -> None: diff --git a/zerver/tests/test_type_debug.py b/zerver/tests/test_type_debug.py index 7c66fc11e3..9bc9fbd6e2 100644 --- a/zerver/tests/test_type_debug.py +++ b/zerver/tests/test_type_debug.py @@ -1,11 +1,10 @@ import sys -from unittest import TestCase from io import StringIO +from typing import Any, Callable, Dict, Iterable, List, Tuple, TypeVar +from unittest import TestCase from zerver.lib.type_debug import print_types -from typing import Any, Callable, Dict, Iterable, Tuple, TypeVar, List - T = TypeVar('T') def add(x: Any=0, y: Any=0) -> Any: diff --git a/zerver/tests/test_typing.py b/zerver/tests/test_typing.py index e85ca32e6a..750b293b7a 100644 --- a/zerver/tests/test_typing.py +++ b/zerver/tests/test_typing.py @@ -1,17 +1,11 @@ -import ujson -from typing import Any, Mapping, List +from typing import Any, List, Mapping + +import ujson + +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import queries_captured, tornado_redirected_to_list +from zerver.models import Huddle, get_huddle_hash -from zerver.lib.test_helpers import ( - tornado_redirected_to_list, - queries_captured, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.models import ( - Huddle, - get_huddle_hash, -) class TypingValidateOperatorTest(ZulipTestCase): def test_missing_parameter(self) -> None: diff --git a/zerver/tests/test_unread.py b/zerver/tests/test_unread.py index a427d4988f..803e1cf739 100644 --- a/zerver/tests/test_unread.py +++ b/zerver/tests/test_unread.py @@ -1,32 +1,15 @@ from typing import Any, List, Mapping +from unittest import mock +import ujson from django.db import connection -from zerver.models import ( - get_realm, - get_stream, - get_user, - Subscription, - UserMessage, - UserProfile, -) - -from zerver.lib.fix_unreads import ( - fix, - fix_pre_pointer, - fix_unsubscribed, -) -from zerver.lib.test_helpers import ( - get_subscription, - tornado_redirected_to_list, -) -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.fix_unreads import fix, fix_pre_pointer, fix_unsubscribed +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import get_subscription, tornado_redirected_to_list from zerver.lib.topic_mutes import add_topic_mute +from zerver.models import Subscription, UserMessage, UserProfile, get_realm, get_stream, get_user -from unittest import mock -import ujson class PointerTest(ZulipTestCase): diff --git a/zerver/tests/test_upload.py b/zerver/tests/test_upload.py index 1e08e1ceef..fa0084658c 100644 --- a/zerver/tests/test_upload.py +++ b/zerver/tests/test_upload.py @@ -1,63 +1,80 @@ -from django.conf import settings -from django.test import TestCase +import datetime +import io +import os +import re +import shutil +import time +import urllib +from io import StringIO +from unittest import mock from unittest.mock import patch -from zerver.lib.avatar import ( - avatar_url, - get_avatar_field, +import botocore.exceptions +import ujson +from django.conf import settings +from django.test import TestCase +from django.utils.timezone import now as timezone_now +from django_sendfile.sendfile import _get_sendfile +from PIL import Image + +import zerver.lib.upload +from scripts.lib.zulip_tools import get_dev_uuid_var_path +from zerver.lib.actions import ( + do_change_icon_source, + do_change_logo_source, + do_change_plan_type, + do_delete_old_unclaimed_attachments, + do_set_realm_property, + internal_send_private_message, ) +from zerver.lib.avatar import avatar_url, get_avatar_field from zerver.lib.avatar_hash import user_avatar_path +from zerver.lib.cache import cache_get, get_realm_used_upload_space_cache_key +from zerver.lib.create_user import copy_user_settings from zerver.lib.initial_password import initial_password from zerver.lib.realm_icon import realm_icon_url from zerver.lib.realm_logo import get_realm_logo_url -from zerver.lib.test_classes import ZulipTestCase, UploadSerializeMixin +from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase from zerver.lib.test_helpers import ( avatar_disk_path, - get_test_image_file, - use_s3_backend, create_s3_buckets, + get_test_image_file, queries_captured, + use_s3_backend, ) -from zerver.lib.upload import sanitize_name, S3UploadBackend, \ - upload_message_file, upload_emoji_image, delete_message_image, LocalUploadBackend, \ - ZulipUploadBackend, MEDIUM_AVATAR_SIZE, resize_avatar, \ - resize_emoji, BadImageError, get_realm_for_filename, \ - DEFAULT_AVATAR_SIZE, DEFAULT_EMOJI_SIZE, exif_rotate, \ - upload_export_tarball, delete_export_tarball -import zerver.lib.upload -from zerver.models import Attachment, get_user_by_delivery_email, \ - Message, UserProfile, Realm, \ - RealmDomain, RealmEmoji, get_realm, get_system_bot, \ - validate_attachment_request -from zerver.lib.actions import ( - do_change_plan_type, - do_change_icon_source, - do_change_logo_source, - do_delete_old_unclaimed_attachments, - internal_send_private_message, - do_set_realm_property, +from zerver.lib.upload import ( + DEFAULT_AVATAR_SIZE, + DEFAULT_EMOJI_SIZE, + MEDIUM_AVATAR_SIZE, + BadImageError, + LocalUploadBackend, + S3UploadBackend, + ZulipUploadBackend, + delete_export_tarball, + delete_message_image, + exif_rotate, + get_realm_for_filename, + resize_avatar, + resize_emoji, + sanitize_name, + upload_emoji_image, + upload_export_tarball, + upload_message_file, ) -from zerver.lib.cache import get_realm_used_upload_space_cache_key, cache_get -from zerver.lib.create_user import copy_user_settings from zerver.lib.users import get_api_key +from zerver.models import ( + Attachment, + Message, + Realm, + RealmDomain, + RealmEmoji, + UserProfile, + get_realm, + get_system_bot, + get_user_by_delivery_email, + validate_attachment_request, +) -from scripts.lib.zulip_tools import get_dev_uuid_var_path - -import urllib -import ujson -from PIL import Image -import botocore.exceptions - -from io import StringIO -from unittest import mock -import os -import io -import shutil -import re -import time -import datetime -from django.utils.timezone import now as timezone_now -from django_sendfile.sendfile import _get_sendfile def destroy_uploads() -> None: if os.path.exists(settings.LOCAL_UPLOADS_DIR): diff --git a/zerver/tests/test_urls.py b/zerver/tests/test_urls.py index 60b4555fa0..10a2de74f9 100644 --- a/zerver/tests/test_urls.py +++ b/zerver/tests/test_urls.py @@ -1,16 +1,17 @@ import importlib import os -import ujson +from typing import List, Optional import django.urls.resolvers -from django.test import TestCase, Client -from typing import List, Optional +import ujson +from django.test import Client, TestCase from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_runner import slow from zerver.models import Stream from zproject import urls + class PublicURLTest(ZulipTestCase): """ Account creation URLs are accessible even when not logged in. Authenticated diff --git a/zerver/tests/test_user_groups.py b/zerver/tests/test_user_groups.py index 423a533dcb..3ac492c780 100644 --- a/zerver/tests/test_user_groups.py +++ b/zerver/tests/test_user_groups.py @@ -1,26 +1,21 @@ -import ujson from unittest import mock -from zerver.lib.actions import ( - ensure_stream, -) +import ujson +from zerver.lib.actions import do_set_realm_property, ensure_stream from zerver.lib.test_classes import ZulipTestCase -from zerver.lib.test_helpers import ( - most_recent_usermessage, -) -from zerver.lib.actions import do_set_realm_property +from zerver.lib.test_helpers import most_recent_usermessage from zerver.lib.user_groups import ( check_add_user_to_user_group, check_remove_user_from_user_group, create_user_group, + get_memberships_of_users, get_user_groups, user_groups_in_realm, - get_memberships_of_users, user_groups_in_realm_serialized, ) -from zerver.models import UserGroup, get_realm, Realm, \ - UserGroupMembership +from zerver.models import Realm, UserGroup, UserGroupMembership, get_realm + class UserGroupTestCase(ZulipTestCase): def create_user_group_for_test(self, group_name: str, diff --git a/zerver/tests/test_user_status.py b/zerver/tests/test_user_status.py index 051b7606c1..5c178c64f7 100644 --- a/zerver/tests/test_user_status.py +++ b/zerver/tests/test_user_status.py @@ -1,24 +1,12 @@ +from typing import Any, Dict, Set + import ujson -from zerver.lib.test_classes import ( - ZulipTestCase, -) -from zerver.lib.test_helpers import ( - EventInfo, - capture_event, -) -from zerver.lib.user_status import ( - get_user_info_dict, - update_user_status, -) +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.test_helpers import EventInfo, capture_event +from zerver.lib.user_status import get_user_info_dict, update_user_status +from zerver.models import UserProfile, UserStatus, get_client -from zerver.models import ( - get_client, - UserProfile, - UserStatus, -) - -from typing import Any, Dict, Set def get_away_user_ids(realm_id: int) -> Set[int]: user_dict = get_user_info_dict(realm_id) diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index cc4a7278e7..cf9213d117 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -1,8 +1,29 @@ +import datetime from email.utils import parseaddr +from typing import Any, Dict, Iterable, List, Mapping, Optional, TypeVar, Union +from unittest import mock -from typing import (Any, Dict, Iterable, List, Mapping, - Optional, TypeVar, Union) +import ujson +from django.conf import settings +from django.test import override_settings +from zerver.lib.actions import ( + create_users, + do_change_user_role, + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_set_realm_property, + get_emails_from_user_ids, + get_recipient_info, +) +from zerver.lib.avatar import avatar_url, get_gravatar_url +from zerver.lib.create_user import copy_user_settings +from zerver.lib.events import do_events_register +from zerver.lib.exceptions import JsonableError +from zerver.lib.send_email import clear_scheduled_emails, deliver_email, send_future_email +from zerver.lib.stream_topic import StreamTopicTarget +from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( get_subscription, most_recent_message, @@ -11,45 +32,28 @@ from zerver.lib.test_helpers import ( simulated_empty_cache, tornado_redirected_to_list, ) -from zerver.lib.test_classes import ( - ZulipTestCase, -) - -from zerver.models import UserProfile, Recipient, Realm, \ - RealmDomain, UserHotspot, get_client, \ - get_user, get_user_by_delivery_email, get_realm, get_stream, \ - get_source_profile, get_system_bot, \ - ScheduledEmail, check_valid_user_ids, \ - get_user_by_id_in_realm_including_cross_realm, CustomProfileField, \ - InvalidFakeEmailDomain, get_fake_email_domain - -from zerver.lib.avatar import avatar_url, get_gravatar_url -from zerver.lib.exceptions import JsonableError -from zerver.lib.send_email import send_future_email, clear_scheduled_emails, \ - deliver_email -from zerver.lib.actions import ( - create_users, - get_emails_from_user_ids, - get_recipient_info, - do_deactivate_user, - do_reactivate_user, - do_change_user_role, - do_create_user, - do_set_realm_property, -) -from zerver.lib.create_user import copy_user_settings -from zerver.lib.events import do_events_register from zerver.lib.topic_mutes import add_topic_mute -from zerver.lib.stream_topic import StreamTopicTarget -from zerver.lib.users import user_ids_to_users, access_user_by_id, \ - get_accounts_for_email - -from django.conf import settings -from django.test import override_settings - -import datetime -from unittest import mock -import ujson +from zerver.lib.users import access_user_by_id, get_accounts_for_email, user_ids_to_users +from zerver.models import ( + CustomProfileField, + InvalidFakeEmailDomain, + Realm, + RealmDomain, + Recipient, + ScheduledEmail, + UserHotspot, + UserProfile, + check_valid_user_ids, + get_client, + get_fake_email_domain, + get_realm, + get_source_profile, + get_stream, + get_system_bot, + get_user, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, +) K = TypeVar('K') V = TypeVar('V') diff --git a/zerver/tests/test_webhooks_common.py b/zerver/tests/test_webhooks_common.py index cf4b3ebeeb..a842b230b1 100644 --- a/zerver/tests/test_webhooks_common.py +++ b/zerver/tests/test_webhooks_common.py @@ -1,20 +1,24 @@ from types import SimpleNamespace -from unittest.mock import MagicMock, patch from typing import Dict +from unittest.mock import MagicMock, patch from django.http import HttpRequest from zerver.decorator import api_key_only_webhook_view from zerver.lib.exceptions import InvalidJSONError, JsonableError -from zerver.lib.test_classes import ZulipTestCase, WebhookTestCase -from zerver.lib.webhooks.common import \ - validate_extract_webhook_http_header, \ - MISSING_EVENT_HEADER_MESSAGE, MissingHTTPEventHeader, \ - INVALID_JSON_MESSAGE, get_fixture_http_headers, standardize_headers -from zerver.models import get_user, get_realm, UserProfile -from zerver.lib.users import get_api_key from zerver.lib.send_email import FromAddress +from zerver.lib.test_classes import WebhookTestCase, ZulipTestCase from zerver.lib.test_helpers import HostRequestMock +from zerver.lib.users import get_api_key +from zerver.lib.webhooks.common import ( + INVALID_JSON_MESSAGE, + MISSING_EVENT_HEADER_MESSAGE, + MissingHTTPEventHeader, + get_fixture_http_headers, + standardize_headers, + validate_extract_webhook_http_header, +) +from zerver.models import UserProfile, get_realm, get_user class WebhooksCommonTestCase(ZulipTestCase): diff --git a/zerver/tests/test_widgets.py b/zerver/tests/test_widgets.py index 9218277933..d362930c30 100644 --- a/zerver/tests/test_widgets.py +++ b/zerver/tests/test_widgets.py @@ -1,14 +1,12 @@ +from typing import Any, Dict + import ujson -from typing import Dict, Any - +from zerver.lib.test_classes import ZulipTestCase +from zerver.lib.validator import check_widget_content +from zerver.lib.widget import get_widget_data from zerver.models import SubMessage -from zerver.lib.test_classes import ZulipTestCase - -from zerver.lib.widget import get_widget_data - -from zerver.lib.validator import check_widget_content class WidgetContentTestCase(ZulipTestCase): def test_validation(self) -> None: diff --git a/zerver/tests/test_zcommand.py b/zerver/tests/test_zcommand.py index 31165272e2..08c1ca7d16 100644 --- a/zerver/tests/test_zcommand.py +++ b/zerver/tests/test_zcommand.py @@ -1,6 +1,5 @@ -from zerver.lib.test_classes import ( - ZulipTestCase, -) +from zerver.lib.test_classes import ZulipTestCase + class ZcommandTest(ZulipTestCase): diff --git a/zerver/tests/test_zephyr.py b/zerver/tests/test_zephyr.py index fef809b576..dae495d455 100644 --- a/zerver/tests/test_zephyr.py +++ b/zerver/tests/test_zephyr.py @@ -1,12 +1,12 @@ -import ujson - -from django.http import HttpResponse -from unittest.mock import patch from typing import Any +from unittest.mock import patch + +import ujson +from django.http import HttpResponse from zerver.lib.test_classes import ZulipTestCase from zerver.lib.users import get_api_key -from zerver.models import get_user, get_realm +from zerver.models import get_realm, get_user class ZephyrTest(ZulipTestCase): diff --git a/zerver/tornado/application.py b/zerver/tornado/application.py index 019cffbf23..a303b9a15f 100644 --- a/zerver/tornado/application.py +++ b/zerver/tornado/application.py @@ -2,11 +2,12 @@ import atexit import tornado.web from django.conf import settings -from zerver.tornado import autoreload from zerver.lib.queue import get_queue_client +from zerver.tornado import autoreload from zerver.tornado.handlers import AsyncDjangoHandler + def setup_tornado_rabbitmq() -> None: # nocoverage # When tornado is shut down, disconnect cleanly from rabbitmq if settings.USING_RABBITMQ: diff --git a/zerver/tornado/autoreload.py b/zerver/tornado/autoreload.py index a9d7a98e54..4aaec8f387 100644 --- a/zerver/tornado/autoreload.py +++ b/zerver/tornado/autoreload.py @@ -50,18 +50,17 @@ incorrectly. # code that didn't work. -import os -import sys import functools import importlib +import os +import subprocess +import sys import traceback import types -import subprocess import weakref -from tornado import ioloop +from tornado import ioloop, process from tornado.log import gen_log -from tornado import process try: import signal diff --git a/zerver/tornado/descriptors.py b/zerver/tornado/descriptors.py index 135ead63bc..402067a935 100644 --- a/zerver/tornado/descriptors.py +++ b/zerver/tornado/descriptors.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: from zerver.tornado.event_queue import ClientDescriptor diff --git a/zerver/tornado/event_queue.py b/zerver/tornado/event_queue.py index 4803ff7c40..2aea57b176 100644 --- a/zerver/tornado/event_queue.py +++ b/zerver/tornado/event_queue.py @@ -1,39 +1,57 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/events-system.html for # high-level documentation on how this system works. -from typing import cast, AbstractSet, Any, Callable, Deque, Dict, List, \ - Mapping, MutableMapping, Optional, Iterable, Sequence, Set, Union +import atexit +import copy +import logging +import os +import random +import signal +import sys +import time +import traceback +from collections import deque +from typing import ( + AbstractSet, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + Union, + cast, +) + +import requests +import tornado.ioloop +import ujson +from django.conf import settings +from django.utils.translation import ugettext as _ from typing_extensions import TypedDict -from django.utils.translation import ugettext as _ -from django.conf import settings -from collections import deque -import os -import time -import logging -import ujson -import requests -import atexit -import sys -import signal -import traceback -import tornado.ioloop -import random -from zerver.models import UserProfile, Client, Realm from zerver.decorator import cachify -from zerver.tornado.handlers import clear_handler_by_id, get_handler_by_id, \ - finish_handler, handler_stats_string -from zerver.lib.utils import statsd -from zerver.middleware import async_request_timer_restart from zerver.lib.message import MessageDict from zerver.lib.narrow import build_narrow_filter from zerver.lib.queue import queue_json_publish, retry_event from zerver.lib.request import JsonableError +from zerver.lib.utils import statsd +from zerver.middleware import async_request_timer_restart +from zerver.models import Client, Realm, UserProfile +from zerver.tornado.autoreload import add_reload_hook from zerver.tornado.descriptors import clear_descriptor_by_handler_id, set_descriptor_by_handler_id from zerver.tornado.exceptions import BadEventQueueIdError -from zerver.tornado.sharding import get_tornado_uri, get_tornado_port, \ - notify_tornado_queue_name -from zerver.tornado.autoreload import add_reload_hook -import copy +from zerver.tornado.handlers import ( + clear_handler_by_id, + finish_handler, + get_handler_by_id, + handler_stats_string, +) +from zerver.tornado.sharding import get_tornado_port, get_tornado_uri, notify_tornado_queue_name requests_client = requests.Session() for host in ['127.0.0.1', 'localhost']: diff --git a/zerver/tornado/exceptions.py b/zerver/tornado/exceptions.py index 652dc579b6..9b194e6d78 100644 --- a/zerver/tornado/exceptions.py +++ b/zerver/tornado/exceptions.py @@ -2,6 +2,7 @@ from django.utils.translation import ugettext as _ from zerver.lib.exceptions import ErrorCode, JsonableError + class BadEventQueueIdError(JsonableError): code = ErrorCode.BAD_EVENT_QUEUE_ID data_fields = ['queue_id'] diff --git a/zerver/tornado/handlers.py b/zerver/tornado/handlers.py index e1af1f8a5c..74659ca757 100644 --- a/zerver/tornado/handlers.py +++ b/zerver/tornado/handlers.py @@ -7,8 +7,8 @@ from django import http from django.core import signals from django.core.handlers import base from django.core.handlers.wsgi import WSGIRequest, get_script_name -from django.urls import set_script_prefix from django.http import HttpRequest, HttpResponse +from django.urls import set_script_prefix from tornado.wsgi import WSGIContainer from zerver.lib.response import json_response diff --git a/zerver/tornado/sharding.py b/zerver/tornado/sharding.py index aeefb66738..7a7cb9588b 100644 --- a/zerver/tornado/sharding.py +++ b/zerver/tornado/sharding.py @@ -1,8 +1,10 @@ +import json +import os + from django.conf import settings from zerver.models import Realm -import json -import os + shard_map = {} if os.path.exists("/etc/zulip/sharding.json"): with open("/etc/zulip/sharding.json") as f: diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index fb8a24e9f7..9de1c1d3f2 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -5,16 +5,20 @@ import ujson from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, has_request_variables, internal_notify_view, \ - process_client +from zerver.decorator import REQ, has_request_variables, internal_notify_view, process_client from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_bool, check_int, check_list, check_string, \ - to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_int, + check_list, + check_string, + to_non_negative_int, +) from zerver.models import Client, UserProfile, get_client, get_user_profile_by_id -from zerver.tornado.event_queue import fetch_events, \ - get_client_descriptor, process_notification -from zerver.tornado.handlers import AsyncDjangoHandler +from zerver.tornado.event_queue import fetch_events, get_client_descriptor, process_notification from zerver.tornado.exceptions import BadEventQueueIdError +from zerver.tornado.handlers import AsyncDjangoHandler + @internal_notify_view(True) def notify(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/alert_words.py b/zerver/views/alert_words.py index c4821fd847..1f6ba2d23a 100644 --- a/zerver/views/alert_words.py +++ b/zerver/views/alert_words.py @@ -1,14 +1,14 @@ -from django.http import HttpResponse, HttpRequest - from typing import List -from zerver.models import UserProfile -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success -from zerver.lib.validator import check_list, check_string, check_capped_string +from django.http import HttpRequest, HttpResponse from zerver.lib.actions import do_add_alert_words, do_remove_alert_words from zerver.lib.alert_words import user_alert_words +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success +from zerver.lib.validator import check_capped_string, check_list, check_string +from zerver.models import UserProfile + def list_alert_words(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'alert_words': user_alert_words(user_profile)}) diff --git a/zerver/views/archive.py b/zerver/views/archive.py index 3b610a17fe..fbb7cbad18 100644 --- a/zerver/views/archive.py +++ b/zerver/views/archive.py @@ -3,17 +3,15 @@ from typing import List, Optional from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.template import loader -from zerver.lib.streams import get_stream_by_id -from zerver.models import Message, UserProfile from zerver.lib.avatar import get_gravatar_url -from zerver.lib.response import json_success -from zerver.lib.timestamp import datetime_to_timestamp -from zerver.lib.topic import ( - get_topic_history_for_web_public_stream, - messages_for_topic, -) from zerver.lib.exceptions import JsonableError +from zerver.lib.response import json_success +from zerver.lib.streams import get_stream_by_id +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.topic import get_topic_history_for_web_public_stream, messages_for_topic +from zerver.models import Message, UserProfile + def archive(request: HttpRequest, stream_id: int, diff --git a/zerver/views/attachments.py b/zerver/views/attachments.py index 250c004b87..e6a6cace8b 100644 --- a/zerver/views/attachments.py +++ b/zerver/views/attachments.py @@ -1,10 +1,9 @@ from django.http import HttpRequest, HttpResponse -from zerver.models import UserProfile from zerver.lib.actions import notify_attachment_update +from zerver.lib.attachments import access_attachment_by_id, remove_attachment, user_attachments from zerver.lib.response import json_success -from zerver.lib.attachments import user_attachments, remove_attachment, \ - access_attachment_by_id +from zerver.models import UserProfile def list_by_user(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/auth.py b/zerver/views/auth.py index 1fbe546f9f..86fc3148bc 100644 --- a/zerver/views/auth.py +++ b/zerver/views/auth.py @@ -1,39 +1,47 @@ +import logging import os +import urllib +from functools import wraps +from typing import Any, Dict, List, Mapping, Optional, cast + +import jwt from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from django.forms import Form from django.conf import settings from django.contrib.auth import authenticate -from django.contrib.auth.views import LoginView as DjangoLoginView, \ - logout_then_login as django_logout_then_login +from django.contrib.auth.views import LoginView as DjangoLoginView from django.contrib.auth.views import PasswordResetView as DjangoPasswordResetView -from django.urls import reverse -from zerver.decorator import require_post, \ - process_client, do_login, log_view_func -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, \ - HttpResponseServerError -from django.template.response import SimpleTemplateResponse +from django.contrib.auth.views import logout_then_login as django_logout_then_login +from django.forms import Form +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect, HttpResponseServerError from django.shortcuts import redirect, render +from django.template.response import SimpleTemplateResponse +from django.urls import reverse +from django.utils.http import is_safe_url +from django.utils.translation import ugettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_safe from django.views.generic import TemplateView -from django.utils.translation import ugettext as _ -from django.utils.http import is_safe_url -from functools import wraps -import urllib -from typing import Any, Dict, List, Optional, Mapping, cast +from social_django.utils import load_backend, load_strategy +from two_factor.forms import BackupTokenForm +from two_factor.views import LoginView as BaseTwoFactorLoginView from confirmation.models import Confirmation, create_confirmation_link -from zerver.context_processors import zulip_default_context, get_realm_from_request, \ - login_context -from zerver.forms import HomepageForm, OurAuthenticationForm, \ - DEACTIVATED_ACCOUNT_ERROR, ZulipPasswordResetForm, \ - AuthenticationTokenForm +from version import API_FEATURE_LEVEL, ZULIP_VERSION +from zerver.context_processors import get_realm_from_request, login_context, zulip_default_context +from zerver.decorator import do_login, log_view_func, process_client, require_post +from zerver.forms import ( + DEACTIVATED_ACCOUNT_ERROR, + AuthenticationTokenForm, + HomepageForm, + OurAuthenticationForm, + ZulipPasswordResetForm, +) from zerver.lib.mobile_auth_otp import otp_encrypt_api_key from zerver.lib.push_notifications import push_notifications_enabled from zerver.lib.pysa import mark_sanitized from zerver.lib.realm_icon import realm_icon_url -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.sessions import set_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_subdomain_root_or_alias from zerver.lib.types import ViewFuncT @@ -42,23 +50,24 @@ from zerver.lib.user_agent import parse_user_agent from zerver.lib.users import get_api_key from zerver.lib.utils import has_api_key_format from zerver.lib.validator import validate_login_email -from zerver.models import PreregistrationUser, UserProfile, remote_user_to_email, Realm, \ - get_realm +from zerver.models import PreregistrationUser, Realm, UserProfile, get_realm, remote_user_to_email from zerver.signals import email_on_new_login -from zproject.backends import password_auth_enabled, dev_auth_enabled, \ - ldap_auth_enabled, ZulipLDAPConfigurationError, ZulipLDAPAuthBackend, \ - AUTH_BACKEND_NAME_MAP, auth_enabled_helper, saml_auth_enabled, SAMLAuthBackend, \ - redirect_to_config_error, ZulipRemoteUserBackend, validate_otp_params, ExternalAuthResult, \ - ExternalAuthDataDict -from version import ZULIP_VERSION, API_FEATURE_LEVEL - -import jwt -import logging - -from social_django.utils import load_backend, load_strategy - -from two_factor.forms import BackupTokenForm -from two_factor.views import LoginView as BaseTwoFactorLoginView +from zproject.backends import ( + AUTH_BACKEND_NAME_MAP, + ExternalAuthDataDict, + ExternalAuthResult, + SAMLAuthBackend, + ZulipLDAPAuthBackend, + ZulipLDAPConfigurationError, + ZulipRemoteUserBackend, + auth_enabled_helper, + dev_auth_enabled, + ldap_auth_enabled, + password_auth_enabled, + redirect_to_config_error, + saml_auth_enabled, + validate_otp_params, +) ExtraContext = Optional[Dict[str, Any]] diff --git a/zerver/views/camo.py b/zerver/views/camo.py index fecbdd868e..414eb6f602 100644 --- a/zerver/views/camo.py +++ b/zerver/views/camo.py @@ -1,13 +1,13 @@ +import binascii + from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import ( - HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound, -) + from zerver.lib.camo import is_camo_url_valid from zerver.lib.thumbnail import generate_thumbnail_url -import binascii def handle_camo_url(request: HttpRequest, digest: str, received_url: str) -> HttpResponse: diff --git a/zerver/views/compatibility.py b/zerver/views/compatibility.py index 6c0e08b158..55b9d4890e 100644 --- a/zerver/views/compatibility.py +++ b/zerver/views/compatibility.py @@ -1,13 +1,14 @@ -from django.http import HttpResponse, HttpRequest import re from typing import List, Optional, Tuple +from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ +from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION from zerver.lib.response import json_error, json_success from zerver.lib.user_agent import parse_user_agent from zerver.signals import get_device_browser -from version import DESKTOP_MINIMUM_VERSION, DESKTOP_WARNING_VERSION + def pop_numerals(ver: str) -> Tuple[List[int], str]: match = re.search(r'^( \d+ (?: \. \d+ )* ) (.*)', ver, re.X) diff --git a/zerver/views/custom_profile_fields.py b/zerver/views/custom_profile_fields.py index 850a11165f..96941837ef 100644 --- a/zerver/views/custom_profile_fields.py +++ b/zerver/views/custom_profile_fields.py @@ -1,29 +1,35 @@ -from typing import Union, List, Dict -import ujson +from typing import Dict, List, Union +import ujson from django.db import IntegrityError from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, human_users_only -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.actions import (try_add_realm_custom_profile_field, - do_remove_realm_custom_profile_field, - try_update_realm_custom_profile_field, - do_update_user_custom_profile_data_if_changed, - try_reorder_realm_custom_profile_fields, - try_add_realm_default_custom_profile_field, - check_remove_custom_profile_field_value) -from zerver.lib.response import json_success, json_error -from zerver.lib.types import ProfileFieldData -from zerver.lib.validator import (check_dict, check_list, check_int, - validate_choice_field_data, check_capped_string) - -from zerver.models import (UserProfile, - CustomProfileField, custom_profile_fields_for_realm) +from zerver.decorator import human_users_only, require_realm_admin +from zerver.lib.actions import ( + check_remove_custom_profile_field_value, + do_remove_realm_custom_profile_field, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, + try_reorder_realm_custom_profile_fields, + try_update_realm_custom_profile_field, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.users import validate_user_custom_profile_data from zerver.lib.external_accounts import validate_external_account_field_data +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.types import ProfileFieldData +from zerver.lib.users import validate_user_custom_profile_data +from zerver.lib.validator import ( + check_capped_string, + check_dict, + check_int, + check_list, + validate_choice_field_data, +) +from zerver.models import CustomProfileField, UserProfile, custom_profile_fields_for_realm + def list_realm_custom_profile_fields(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: fields = custom_profile_fields_for_realm(user_profile.realm_id) diff --git a/zerver/views/development/email_log.py b/zerver/views/development/email_log.py index 3cab8c4c29..bbfcbff344 100755 --- a/zerver/views/development/email_log.py +++ b/zerver/views/development/email_log.py @@ -1,25 +1,19 @@ -from django.conf import settings -from django.http import HttpRequest, HttpResponse -from django.shortcuts import render, redirect -from django.views.decorators.http import require_safe - -from zerver.models import ( - get_realm, get_user_by_delivery_email, get_realm_stream, Realm, -) -from zerver.lib.email_notifications import enqueue_welcome_emails -from zerver.lib.response import json_success -from zerver.lib.actions import do_change_user_delivery_email, \ - do_send_realm_reactivation_email -from zproject.email_backends import ( - get_forward_address, - set_forward_address, -) -import urllib -from confirmation.models import Confirmation, confirmation_url - import os import subprocess +import urllib + import ujson +from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect, render +from django.views.decorators.http import require_safe + +from confirmation.models import Confirmation, confirmation_url +from zerver.lib.actions import do_change_user_delivery_email, do_send_realm_reactivation_email +from zerver.lib.email_notifications import enqueue_welcome_emails +from zerver.lib.response import json_success +from zerver.models import Realm, get_realm, get_realm_stream, get_user_by_delivery_email +from zproject.email_backends import get_forward_address, set_forward_address ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') diff --git a/zerver/views/development/integrations.py b/zerver/views/development/integrations.py index 9da34f13f9..0b5701338a 100644 --- a/zerver/views/development/integrations.py +++ b/zerver/views/development/integrations.py @@ -1,19 +1,17 @@ import os -import ujson from typing import Any, Dict, List, Optional +import ujson from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.test import Client from zerver.lib.integrations import WEBHOOK_INTEGRATIONS -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import UserProfile, get_realm +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool -from zerver.lib.webhooks.common import get_fixture_http_headers, \ - standardize_headers - +from zerver.lib.webhooks.common import get_fixture_http_headers, standardize_headers +from zerver.models import UserProfile, get_realm ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') diff --git a/zerver/views/development/registration.py b/zerver/views/development/registration.py index ee8a8047ba..467ea77605 100644 --- a/zerver/views/development/registration.py +++ b/zerver/views/development/registration.py @@ -1,14 +1,13 @@ +from typing import Any + from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from confirmation.models import Confirmation, create_confirmation_link - -from typing import Any - -from zerver.models import UserProfile from zerver.lib.response import json_success from zerver.lib.subdomains import get_subdomain +from zerver.models import UserProfile from zerver.views.auth import create_preregistration_user from zerver.views.registration import accounts_register diff --git a/zerver/views/digest.py b/zerver/views/digest.py index 524755f79c..d9eb84762f 100644 --- a/zerver/views/digest.py +++ b/zerver/views/digest.py @@ -1,12 +1,14 @@ import time +from datetime import timedelta + +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.timezone import now as timezone_now -from django.conf import settings -from zerver.lib.digest import handle_digest_email, DIGEST_CUTOFF from zerver.decorator import zulip_login_required -from datetime import timedelta +from zerver.lib.digest import DIGEST_CUTOFF, handle_digest_email + @zulip_login_required def digest_page(request: HttpRequest) -> HttpResponse: diff --git a/zerver/views/documentation.py b/zerver/views/documentation.py index f26c49b4bd..e93f04ce81 100644 --- a/zerver/views/documentation.py +++ b/zerver/views/documentation.py @@ -1,22 +1,22 @@ -from typing import Any, Dict, Tuple -from collections import OrderedDict -from django.views.generic import TemplateView -from django.conf import settings -from django.http import HttpRequest, HttpResponse, HttpResponseNotFound -from django.template import loader - import os import random import re +from collections import OrderedDict +from typing import Any, Dict, Tuple +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseNotFound +from django.template import loader +from django.views.generic import TemplateView + +from zerver.context_processors import zulip_default_context from zerver.decorator import add_google_analytics_context -from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, \ - WebhookIntegration -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.integrations import CATEGORIES, INTEGRATIONS, HubotIntegration, WebhookIntegration +from zerver.lib.request import REQ, has_request_variables from zerver.lib.subdomains import get_subdomain from zerver.models import Realm from zerver.templatetags.app_filters import render_markdown_path -from zerver.context_processors import zulip_default_context + def add_api_uri_context(context: Dict[str, Any], request: HttpRequest) -> None: context.update(zulip_default_context(request)) diff --git a/zerver/views/email_mirror.py b/zerver/views/email_mirror.py index 8e8260111c..1f04e7f2c6 100644 --- a/zerver/views/email_mirror.py +++ b/zerver/views/email_mirror.py @@ -1,11 +1,11 @@ -import ujson - -from django.http import HttpRequest, HttpResponse from typing import Dict +import ujson +from django.http import HttpRequest, HttpResponse + from zerver.decorator import internal_notify_view from zerver.lib.email_mirror import mirror_email_message -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_string diff --git a/zerver/views/events_register.py b/zerver/views/events_register.py index eab0f35a05..32b3277877 100644 --- a/zerver/views/events_register.py +++ b/zerver/views/events_register.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import Dict, Iterable, Optional, Sequence +from django.http import HttpRequest, HttpResponse + from zerver.lib.events import do_events_register from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.validator import check_dict, check_string, check_list, check_bool +from zerver.lib.validator import check_bool, check_dict, check_list, check_string from zerver.models import Stream, UserProfile + def _default_all_public_streams(user_profile: UserProfile, all_public_streams: Optional[bool]) -> bool: if all_public_streams is not None: diff --git a/zerver/views/home.py b/zerver/views/home.py index 5f15d5ddf8..a072ea2467 100644 --- a/zerver/views/home.py +++ b/zerver/views/home.py @@ -1,35 +1,36 @@ -from typing import Any, List, Dict, Optional, Tuple +import calendar +import logging +import time +from typing import Any, Dict, List, Optional, Tuple from django.conf import settings -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import redirect, render +from django.urls import reverse from django.utils import translation from django.utils.cache import patch_cache_control +from two_factor.utils import default_device from zerver.decorator import zulip_login_required from zerver.forms import ToSForm -from zerver.models import Message, Stream, UserProfile, \ - Realm, PreregistrationUser +from zerver.lib.actions import do_change_tos_version, realm_user_count from zerver.lib.events import do_events_register -from zerver.lib.actions import do_change_tos_version, \ - realm_user_count -from zerver.lib.i18n import get_language_list, get_language_name, \ - get_language_list_for_templates, get_language_translation_data +from zerver.lib.i18n import ( + get_language_list, + get_language_list_for_templates, + get_language_name, + get_language_translation_data, +) from zerver.lib.push_notifications import num_push_devices_for_user from zerver.lib.streams import access_stream_by_name from zerver.lib.subdomains import get_subdomain from zerver.lib.users import compute_show_invites_and_add_streams -from zerver.lib.utils import statsd, generate_random_token -from zerver.views.compatibility import is_outdated_desktop_app, \ - is_unsupported_browser +from zerver.lib.utils import generate_random_token, statsd +from zerver.models import Message, PreregistrationUser, Realm, Stream, UserProfile +from zerver.views.compatibility import is_outdated_desktop_app, is_unsupported_browser from zerver.views.messages import get_latest_update_message_flag_activity from zerver.views.portico import hello_view -from two_factor.utils import default_device -import calendar -import logging -import time def need_accept_tos(user_profile: Optional[UserProfile]) -> bool: if user_profile is None: # nocoverage diff --git a/zerver/views/hotspots.py b/zerver/views/hotspots.py index 7efa693850..929ee2692e 100644 --- a/zerver/views/hotspots.py +++ b/zerver/views/hotspots.py @@ -4,11 +4,12 @@ from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only from zerver.lib.actions import do_mark_hotspot_as_read from zerver.lib.hotspots import ALL_HOTSPOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def mark_hotspot_as_read(request: HttpRequest, user: UserProfile, diff --git a/zerver/views/invite.py b/zerver/views/invite.py index 25e4c5db8d..de93ac5d98 100644 --- a/zerver/views/invite.py +++ b/zerver/views/invite.py @@ -1,19 +1,25 @@ -from django.http import HttpRequest, HttpResponse -from django.utils.translation import ugettext as _ +import re from typing import List, Optional, Set -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.lib.actions import do_invite_users, do_revoke_user_invite, \ - do_revoke_multi_use_invite, do_resend_user_invite_email, \ - do_get_user_invites, do_create_multiuse_invite_link -from zerver.lib.exceptions import OrganizationAdministratorRequired -from zerver.lib.request import REQ, has_request_variables, JsonableError -from zerver.lib.response import json_success, json_error -from zerver.lib.streams import access_stream_by_id -from zerver.lib.validator import check_list, check_int -from zerver.models import PreregistrationUser, Stream, UserProfile, MultiuseInvite +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.lib.actions import ( + do_create_multiuse_invite_link, + do_get_user_invites, + do_invite_users, + do_resend_user_invite_email, + do_revoke_multi_use_invite, + do_revoke_user_invite, +) +from zerver.lib.exceptions import OrganizationAdministratorRequired +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import access_stream_by_id +from zerver.lib.validator import check_int, check_list +from zerver.models import MultiuseInvite, PreregistrationUser, Stream, UserProfile -import re @require_member_or_admin @has_request_variables diff --git a/zerver/views/messages.py b/zerver/views/messages.py index a031d3e2ff..6ad5a5b34b 100644 --- a/zerver/views/messages.py +++ b/zerver/views/messages.py @@ -1,70 +1,119 @@ -from django.utils.translation import ugettext as _ -from django.utils.timezone import now as timezone_now +import datetime +import re +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union, cast + +import ujson +from dateutil.parser import parse as dateparser from django.conf import settings from django.core import validators from django.core.exceptions import ValidationError -from django.db import connection, IntegrityError +from django.db import IntegrityError, connection from django.http import HttpRequest, HttpResponse -from typing import Dict, List, Set, Any, Iterable, \ - Optional, Tuple, Union, Sequence, cast -from zerver.lib.exceptions import JsonableError, ErrorCode -from zerver.lib.html_diff import highlight_html_differences -from zerver.decorator import has_request_variables, REQ from django.utils.html import escape as escape_html +from django.utils.timezone import now as timezone_now +from django.utils.translation import ugettext as _ +from sqlalchemy import func +from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import ( + ColumnElement, + Selectable, + alias, + and_, + column, + join, + literal, + literal_column, + not_, + or_, + select, + table, + union_all, +) + +from zerver.decorator import REQ, has_request_variables from zerver.lib import bugdown -from zerver.lib.zcommand import process_zcommands -from zerver.lib.actions import recipient_for_user_profiles, do_update_message_flags, \ - compute_irc_user_fullname, compute_jabber_user_fullname, \ - create_mirror_user_if_needed, check_send_message, do_update_message, \ - extract_private_recipients, render_incoming_message, do_delete_messages, \ - do_mark_all_as_read, do_mark_stream_messages_as_read, extract_stream_indicator, \ - get_user_info_for_message_updates, check_schedule_message +from zerver.lib.actions import ( + check_schedule_message, + check_send_message, + compute_irc_user_fullname, + compute_jabber_user_fullname, + create_mirror_user_if_needed, + do_delete_messages, + do_mark_all_as_read, + do_mark_stream_messages_as_read, + do_update_message, + do_update_message_flags, + extract_private_recipients, + extract_stream_indicator, + get_user_info_for_message_updates, + recipient_for_user_profiles, + render_incoming_message, +) from zerver.lib.addressee import get_user_profiles, get_user_profiles_by_ids -from zerver.lib.queue import queue_json_publish +from zerver.lib.exceptions import ErrorCode, JsonableError +from zerver.lib.html_diff import highlight_html_differences from zerver.lib.message import ( access_message, + get_first_visible_message_id, messages_for_ids, render_markdown, - get_first_visible_message_id, truncate_body, ) -from zerver.lib.response import json_success, json_error +from zerver.lib.queue import queue_json_publish +from zerver.lib.response import json_error, json_success from zerver.lib.sqlalchemy_utils import get_sqlalchemy_connection -from zerver.lib.streams import access_stream_by_id, get_public_streams_queryset, \ - can_access_stream_history_by_name, can_access_stream_history_by_id, \ - get_stream_by_narrow_operand_access_unchecked, get_stream_by_id -from zerver.lib.timestamp import datetime_to_timestamp, convert_to_UTC +from zerver.lib.streams import ( + access_stream_by_id, + can_access_stream_history_by_id, + can_access_stream_history_by_name, + get_public_streams_queryset, + get_stream_by_id, + get_stream_by_narrow_operand_access_unchecked, +) +from zerver.lib.timestamp import convert_to_UTC, datetime_to_timestamp from zerver.lib.timezone import get_timezone from zerver.lib.topic import ( - topic_column_sa, - topic_match_sa, - user_message_exists_for_topic, DB_TOPIC_NAME, LEGACY_PREV_TOPIC, MATCH_TOPIC, REQ_topic, + topic_column_sa, + topic_match_sa, + user_message_exists_for_topic, ) from zerver.lib.topic_mutes import exclude_topic_mutes from zerver.lib.utils import statsd -from zerver.lib.validator import \ - check_list, check_int, check_dict, check_string, check_bool, \ - check_string_or_int_list, check_string_or_int, check_string_in, \ - check_required_string, to_non_negative_int +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_list, + check_required_string, + check_string, + check_string_in, + check_string_or_int, + check_string_or_int_list, + to_non_negative_int, +) +from zerver.lib.zcommand import process_zcommands from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.models import Message, UserProfile, Stream, Subscription, Client,\ - Realm, RealmDomain, Recipient, UserMessage, UserActivity, \ - email_to_domain, get_realm, get_active_streams, get_user_including_cross_realm, \ - get_user_by_id_in_realm_including_cross_realm - -from sqlalchemy import func -from sqlalchemy.dialects import postgresql -from sqlalchemy.sql import select, join, column, literal_column, literal, and_, \ - or_, not_, union_all, alias, Selectable, ColumnElement, table - -from dateutil.parser import parse as dateparser -import re -import ujson -import datetime +from zerver.models import ( + Client, + Message, + Realm, + RealmDomain, + Recipient, + Stream, + Subscription, + UserActivity, + UserMessage, + UserProfile, + email_to_domain, + get_active_streams, + get_realm, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, +) LARGER_THAN_MAX_MESSAGE_ID = 10000000000000000 MAX_MESSAGES_PER_FETCH = 5000 diff --git a/zerver/views/muting.py b/zerver/views/muting.py index 77dbfdcd75..c4f46c5aee 100644 --- a/zerver/views/muting.py +++ b/zerver/views/muting.py @@ -1,13 +1,13 @@ -from django.http import HttpResponse, HttpRequest -from typing import Optional import datetime +from typing import Optional +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ + from zerver.lib.actions import do_mute_topic, do_unmute_topic -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.lib.topic_mutes import topic_is_muted +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.streams import ( access_stream_by_id, access_stream_by_name, @@ -15,9 +15,11 @@ from zerver.lib.streams import ( access_stream_for_unmute_topic_by_name, check_for_exactly_one_stream_arg, ) +from zerver.lib.topic_mutes import topic_is_muted from zerver.lib.validator import check_int from zerver.models import UserProfile + def mute_topic(user_profile: UserProfile, stream_id: Optional[int], stream_name: Optional[str], diff --git a/zerver/views/pointer.py b/zerver/views/pointer.py index 924382098d..08e9a0756e 100644 --- a/zerver/views/pointer.py +++ b/zerver/views/pointer.py @@ -2,11 +2,12 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.lib.actions import do_update_pointer -from zerver.lib.request import has_request_variables, JsonableError, REQ +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import to_non_negative_int from zerver.models import UserProfile, get_usermessage_by_message_id + def get_pointer_backend(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return json_success({'pointer': user_profile.pointer}) diff --git a/zerver/views/portico.py b/zerver/views/portico.py index b6fdc13706..770dc7eaeb 100644 --- a/zerver/views/portico.py +++ b/zerver/views/portico.py @@ -1,12 +1,13 @@ +import ujson from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.template.response import TemplateResponse -import ujson +from version import LATEST_DESKTOP_VERSION from zerver.context_processors import get_realm_from_request, latest_info_context from zerver.decorator import add_google_analytics, redirect_to_login from zerver.models import Realm -from version import LATEST_DESKTOP_VERSION + @add_google_analytics def apps_view(request: HttpRequest, _: str) -> HttpResponse: diff --git a/zerver/views/presence.py b/zerver/views/presence.py index 76525c7f5a..7d7e23d6be 100644 --- a/zerver/views/presence.py +++ b/zerver/views/presence.py @@ -1,27 +1,20 @@ import datetime - -from django.conf import settings from typing import Any, Dict, Optional +from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.actions import ( - do_update_user_status, - update_user_presence, -) -from zerver.lib.presence import ( - get_presence_response, - get_presence_for_user, -) -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error +from zerver.lib.actions import do_update_user_status, update_user_presence +from zerver.lib.presence import get_presence_for_user, get_presence_response +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.validator import check_bool, check_capped_string -from zerver.models import UserActivity, UserPresence, UserProfile, \ - get_active_user +from zerver.models import UserActivity, UserPresence, UserProfile, get_active_user + def get_presence_backend(request: HttpRequest, user_profile: UserProfile, email: str) -> HttpResponse: diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index 2d4d88e3ad..67083211c7 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -3,12 +3,16 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.decorator import human_users_only -from zerver.lib.push_notifications import add_push_device_token, \ - b64_to_hex, remove_push_device_token -from zerver.lib.request import has_request_variables, REQ, JsonableError +from zerver.lib.push_notifications import ( + add_push_device_token, + b64_to_hex, + remove_push_device_token, +) +from zerver.lib.request import REQ, JsonableError, has_request_variables from zerver.lib.response import json_success from zerver.models import PushDeviceToken, UserProfile + def validate_token(token_str: str, kind: int) -> None: if token_str == '' or len(token_str) > 4096: raise JsonableError(_('Empty or invalid length token')) diff --git a/zerver/views/reactions.py b/zerver/views/reactions.py index 31180adddb..951ad88a4f 100644 --- a/zerver/views/reactions.py +++ b/zerver/views/reactions.py @@ -1,8 +1,9 @@ +from typing import Optional + from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import \ - has_request_variables, REQ +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_reaction, do_remove_reaction from zerver.lib.emoji import check_emoji_request, emoji_name_to_emoji_code from zerver.lib.message import access_message @@ -10,7 +11,6 @@ from zerver.lib.request import JsonableError from zerver.lib.response import json_success from zerver.models import Message, Reaction, UserMessage, UserProfile -from typing import Optional def create_historical_message(user_profile: UserProfile, message: Message) -> None: # Users can see and react to messages sent to streams they diff --git a/zerver/views/realm.py b/zerver/views/realm.py index 40ec58ff1f..3dc1ecb8ef 100644 --- a/zerver/views/realm.py +++ b/zerver/views/realm.py @@ -1,31 +1,40 @@ from typing import Any, Dict, Optional + +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.utils.translation import ugettext as _ -from django.core.exceptions import ValidationError from django.views.decorators.http import require_safe +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key from zerver.decorator import require_realm_admin, require_realm_owner +from zerver.forms import check_subdomain_available as check_subdomain from zerver.lib.actions import ( - do_set_realm_message_editing, - do_set_realm_message_deleting, - do_set_realm_authentication_methods, - do_set_realm_notifications_stream, - do_set_realm_signup_notifications_stream, - do_set_realm_property, do_deactivate_realm, do_reactivate_realm, + do_set_realm_authentication_methods, + do_set_realm_message_deleting, + do_set_realm_message_editing, + do_set_realm_notifications_stream, + do_set_realm_property, + do_set_realm_signup_notifications_stream, ) -from zerver.lib.i18n import get_available_language_codes -from zerver.lib.request import has_request_variables, REQ, JsonableError -from zerver.lib.response import json_success, json_error -from zerver.lib.validator import check_string, check_dict, check_bool, check_int, \ - check_int_in, to_positive_or_allowed_int, to_non_negative_int -from zerver.lib.streams import access_stream_by_id from zerver.lib.domains import validate_domain +from zerver.lib.i18n import get_available_language_codes +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import access_stream_by_id +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_string, + to_non_negative_int, + to_positive_or_allowed_int, +) from zerver.models import Realm, UserProfile -from zerver.forms import check_subdomain_available as check_subdomain -from confirmation.models import get_object_from_key, Confirmation, ConfirmationKeyException + @require_realm_admin @has_request_variables diff --git a/zerver/views/realm_domains.py b/zerver/views/realm_domains.py index 85d014226a..60d682ccb7 100644 --- a/zerver/views/realm_domains.py +++ b/zerver/views/realm_domains.py @@ -3,10 +3,9 @@ from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ from zerver.decorator import require_realm_admin -from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, \ - do_remove_realm_domain +from zerver.lib.actions import do_add_realm_domain, do_change_realm_domain, do_remove_realm_domain from zerver.lib.domains import validate_domain -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_bool, check_string from zerver.models import RealmDomain, UserProfile, get_realm_domains diff --git a/zerver/views/realm_emoji.py b/zerver/views/realm_emoji.py index 526d70f090..0758caceaa 100644 --- a/zerver/views/realm_emoji.py +++ b/zerver/views/realm_emoji.py @@ -2,12 +2,12 @@ from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.models import RealmEmoji, UserProfile -from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name -from zerver.lib.request import JsonableError, REQ, has_request_variables -from zerver.lib.response import json_success, json_error -from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji from zerver.decorator import require_member_or_admin +from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji +from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name +from zerver.lib.request import REQ, JsonableError, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmEmoji, UserProfile def list_emoji(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_export.py b/zerver/views/realm_export.py index 4b7b777fc8..29d7bdcc58 100644 --- a/zerver/views/realm_export.py +++ b/zerver/views/realm_export.py @@ -1,20 +1,19 @@ from datetime import timedelta -from analytics.models import RealmCount - +import ujson from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest +from analytics.models import RealmCount from zerver.decorator import require_realm_admin -from zerver.models import RealmAuditLog, UserProfile +from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.lib.export import get_realm_exports_serialized from zerver.lib.queue import queue_json_publish from zerver.lib.response import json_error, json_success -from zerver.lib.export import get_realm_exports_serialized -from zerver.lib.actions import do_delete_realm_export, notify_realm_export +from zerver.models import RealmAuditLog, UserProfile -import ujson @require_realm_admin def export_realm(request: HttpRequest, user: UserProfile) -> HttpResponse: diff --git a/zerver/views/realm_filters.py b/zerver/views/realm_filters.py index 625ef7a173..366cb66bdb 100644 --- a/zerver/views/realm_filters.py +++ b/zerver/views/realm_filters.py @@ -4,9 +4,9 @@ from django.utils.translation import ugettext as _ from zerver.decorator import require_realm_admin from zerver.lib.actions import do_add_realm_filter, do_remove_realm_filter -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.models import realm_filters_for_realm, UserProfile, RealmFilter +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.models import RealmFilter, UserProfile, realm_filters_for_realm # Custom realm filters diff --git a/zerver/views/realm_icon.py b/zerver/views/realm_icon.py index 66054248dd..70dff5711b 100644 --- a/zerver/views/realm_icon.py +++ b/zerver/views/realm_icon.py @@ -1,7 +1,7 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_icon_source diff --git a/zerver/views/realm_logo.py b/zerver/views/realm_logo.py index b7ab5ef03e..ea09ff8cd0 100644 --- a/zerver/views/realm_logo.py +++ b/zerver/views/realm_logo.py @@ -1,16 +1,16 @@ from django.conf import settings +from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpResponse, HttpRequest -from zerver.lib.validator import check_bool -from zerver.lib.request import REQ, has_request_variables from zerver.decorator import require_realm_admin from zerver.lib.actions import do_change_logo_source from zerver.lib.realm_logo import get_realm_logo_url +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success from zerver.lib.upload import upload_logo_image from zerver.lib.url_encoding import add_query_arg_to_redirect_url +from zerver.lib.validator import check_bool from zerver.models import UserProfile diff --git a/zerver/views/registration.py b/zerver/views/registration.py index eb7dae16fb..290acf4cb4 100644 --- a/zerver/views/registration.py +++ b/zerver/views/registration.py @@ -1,55 +1,92 @@ -from typing import List, Dict, Optional +import logging +import smtplib +import urllib +from typing import Dict, List, Optional -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, get_backends -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponse, HttpRequest -from django.shortcuts import redirect, render -from django.core.exceptions import ValidationError from django.core import validators -from zerver.context_processors import get_realm_from_request, login_context -from zerver.models import UserProfile, Realm, Stream, MultiuseInvite, \ - name_changes_disabled, email_to_username, \ - get_realm, get_user_by_delivery_email, get_default_stream_groups, DisposableEmailError, \ - DomainNotAllowedForRealmError, get_source_profile, EmailContainsPlusError -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress -from zerver.lib.actions import do_change_password, do_change_full_name, \ - do_activate_user, do_create_user, do_create_realm, \ - do_set_user_display_setting, lookup_default_stream_groups, bulk_add_subscriptions -from zerver.forms import RegistrationForm, HomepageForm, RealmCreationForm, \ - FindMyTeamForm, RealmRedirectForm +from django.core.exceptions import ValidationError +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect +from django.shortcuts import redirect, render +from django.urls import reverse +from django.utils.translation import ugettext as _ from django_auth_ldap.backend import LDAPBackend, _LDAPUser -from zerver.decorator import require_post, \ - do_login + +from confirmation import settings as confirmation_settings +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + RealmCreationKey, + create_confirmation_link, + get_object_from_key, + render_confirmation_key_error, + validate_key, +) +from zerver.context_processors import get_realm_from_request, login_context +from zerver.decorator import do_login, require_post +from zerver.forms import ( + FindMyTeamForm, + HomepageForm, + RealmCreationForm, + RealmRedirectForm, + RegistrationForm, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_activate_user, + do_change_full_name, + do_change_password, + do_create_realm, + do_create_user, + do_set_user_display_setting, + lookup_default_stream_groups, +) from zerver.lib.create_user import get_role_for_new_user +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.onboarding import send_initial_realm_messages, setup_realm_internal_bots -from zerver.lib.sessions import get_expirable_session_var from zerver.lib.pysa import mark_sanitized +from zerver.lib.send_email import FromAddress, send_email +from zerver.lib.sessions import get_expirable_session_var from zerver.lib.subdomains import get_subdomain, is_root_domain_available from zerver.lib.timezone import get_all_timezones from zerver.lib.url_encoding import add_query_to_redirect_url from zerver.lib.users import get_accounts_for_email from zerver.lib.zephyr import compute_mit_user_fullname -from zerver.views.auth import create_preregistration_user, redirect_and_log_into_subdomain, \ - redirect_to_deactivation_notice, get_safe_redirect_to, finish_desktop_flow, finish_mobile_flow +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + MultiuseInvite, + Realm, + Stream, + UserProfile, + email_to_username, + get_default_stream_groups, + get_realm, + get_source_profile, + get_user_by_delivery_email, + name_changes_disabled, +) +from zerver.views.auth import ( + create_preregistration_user, + finish_desktop_flow, + finish_mobile_flow, + get_safe_redirect_to, + redirect_and_log_into_subdomain, + redirect_to_deactivation_notice, +) +from zproject.backends import ( + ExternalAuthResult, + ZulipLDAPAuthBackend, + ZulipLDAPExceptionNoMatchingLDAPUser, + any_social_backend_enabled, + email_auth_enabled, + email_belongs_to_ldap, + ldap_auth_enabled, + password_auth_enabled, +) -from zproject.backends import ldap_auth_enabled, password_auth_enabled, \ - ZulipLDAPExceptionNoMatchingLDAPUser, email_auth_enabled, ZulipLDAPAuthBackend, \ - email_belongs_to_ldap, any_social_backend_enabled, ExternalAuthResult - -from confirmation.models import Confirmation, RealmCreationKey, ConfirmationKeyException, \ - validate_key, create_confirmation_link, get_object_from_key, \ - render_confirmation_key_error - -from confirmation import settings as confirmation_settings - -import logging -import smtplib - -import urllib def check_prereg_key_and_redirect(request: HttpRequest, confirmation_key: str) -> HttpResponse: confirmation = Confirmation.objects.filter(confirmation_key=confirmation_key).first() diff --git a/zerver/views/report.py b/zerver/views/report.py index de1d62d549..3e665c58a9 100644 --- a/zerver/views/report.py +++ b/zerver/views/report.py @@ -1,25 +1,24 @@ # System documented in https://zulip.readthedocs.io/en/latest/subsystems/logging.html - +import logging +import subprocess from typing import Any, Dict, Optional from django.conf import settings from django.http import HttpRequest, HttpResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST + from zerver.decorator import human_users_only from zerver.lib.bugdown import privacy_clean_markdown -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success from zerver.lib.queue import queue_json_publish +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_success from zerver.lib.storage import static_path from zerver.lib.unminify import SourceMap from zerver.lib.utils import statsd, statsd_key from zerver.lib.validator import check_bool, check_dict, to_non_negative_int from zerver.models import UserProfile -import subprocess -import logging - js_source_map: Optional[SourceMap] = None # Read the source map information for decoding JavaScript backtraces. diff --git a/zerver/views/storage.py b/zerver/views/storage.py index 23e7214640..3772345cc0 100644 --- a/zerver/views/storage.py +++ b/zerver/views/storage.py @@ -1,17 +1,19 @@ +from typing import Dict, List, Optional + from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.bot_storage import ( - get_bot_storage, - set_bot_storage, - remove_bot_storage, - get_keys_in_bot_storage, StateError, + get_bot_storage, + get_keys_in_bot_storage, + remove_bot_storage, + set_bot_storage, ) -from zerver.decorator import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_dict, check_list, check_string from zerver.models import UserProfile -from typing import Dict, List, Optional @has_request_variables def update_storage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/streams.py b/zerver/views/streams.py index f8513a3372..c8a36a24a4 100644 --- a/zerver/views/streams.py +++ b/zerver/views/streams.py @@ -1,39 +1,79 @@ -from typing import Any, Optional, Tuple, List, Set, Iterable, Mapping, Callable, Dict, \ - Union +from collections import defaultdict +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union -from django.utils.translation import ugettext as _ +import ujson from django.conf import settings from django.db import transaction from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ -from zerver.lib.exceptions import JsonableError, ErrorCode +from zerver.decorator import ( + authenticated_json_post_view, + require_non_guest_user, + require_realm_admin, +) +from zerver.lib.actions import ( + bulk_add_subscriptions, + bulk_remove_subscriptions, + do_add_default_stream, + do_add_streams_to_default_stream_group, + do_change_default_stream_group_description, + do_change_default_stream_group_name, + do_change_stream_description, + do_change_stream_invite_only, + do_change_stream_post_policy, + do_change_subscription_property, + do_create_default_stream_group, + do_deactivate_stream, + do_delete_messages, + do_get_streams, + do_remove_default_stream, + do_remove_default_stream_group, + do_remove_streams_from_default_stream_group, + do_rename_stream, + do_send_messages, + gather_subscriptions, + get_subscriber_emails, + internal_prep_private_message, + internal_prep_stream_message, +) +from zerver.lib.exceptions import ErrorCode, JsonableError from zerver.lib.request import REQ, has_request_variables -from zerver.decorator import authenticated_json_post_view, \ - require_realm_admin, require_non_guest_user -from zerver.lib.actions import bulk_remove_subscriptions, \ - do_change_subscription_property, internal_prep_private_message, \ - internal_prep_stream_message, \ - gather_subscriptions, \ - bulk_add_subscriptions, do_send_messages, get_subscriber_emails, do_rename_stream, \ - do_deactivate_stream, do_change_stream_invite_only, do_add_default_stream, \ - do_change_stream_description, do_get_streams, \ - do_remove_default_stream, do_change_stream_post_policy, do_delete_messages, \ - do_create_default_stream_group, do_add_streams_to_default_stream_group, \ - do_remove_streams_from_default_stream_group, do_remove_default_stream_group, \ - do_change_default_stream_group_description, do_change_default_stream_group_name -from zerver.lib.response import json_success, json_error -from zerver.lib.streams import access_stream_by_id, access_stream_by_name, \ - check_stream_name, check_stream_name_available, filter_stream_authorization, \ - list_to_streams, access_stream_for_delete_or_update, access_default_stream_group_by_id +from zerver.lib.response import json_error, json_success +from zerver.lib.streams import ( + access_default_stream_group_by_id, + access_stream_by_id, + access_stream_by_name, + access_stream_for_delete_or_update, + check_stream_name, + check_stream_name_available, + filter_stream_authorization, + list_to_streams, +) from zerver.lib.topic import get_topic_history_for_stream, messages_for_topic -from zerver.lib.validator import check_string, check_int, check_list, check_dict, \ - check_bool, check_variable_type, check_capped_string, check_color, check_dict_only, \ - check_int_in, to_non_negative_int -from zerver.models import UserProfile, Stream, Realm, UserMessage, \ - get_system_bot, get_active_user, get_active_user_profile_by_id_in_realm +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_color, + check_dict, + check_dict_only, + check_int, + check_int_in, + check_list, + check_string, + check_variable_type, + to_non_negative_int, +) +from zerver.models import ( + Realm, + Stream, + UserMessage, + UserProfile, + get_active_user, + get_active_user_profile_by_id_in_realm, + get_system_bot, +) -from collections import defaultdict -import ujson class PrincipalError(JsonableError): code = ErrorCode.UNAUTHORIZED_PRINCIPAL diff --git a/zerver/views/submessage.py b/zerver/views/submessage.py index 014cb80b90..0c0346949a 100644 --- a/zerver/views/submessage.py +++ b/zerver/views/submessage.py @@ -1,21 +1,15 @@ import ujson - from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import ( - has_request_variables, - REQ, -) +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import do_add_submessage from zerver.lib.message import access_message +from zerver.lib.response import json_error, json_success from zerver.lib.validator import check_int -from zerver.lib.response import ( - json_error, - json_success, -) from zerver.models import UserProfile + @has_request_variables def process_submessage(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/thumbnail.py b/zerver/views/thumbnail.py index 2156d07720..7d5bdf2ff5 100644 --- a/zerver/views/thumbnail.py +++ b/zerver/views/thumbnail.py @@ -1,11 +1,14 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html +from typing import Optional + +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden from django.shortcuts import redirect from django.utils.translation import ugettext as _ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden -from typing import Optional -from zerver.models import UserProfile, validate_attachment_request -from zerver.lib.request import has_request_variables, REQ + +from zerver.lib.request import REQ, has_request_variables from zerver.lib.thumbnail import generate_thumbnail_url +from zerver.models import UserProfile, validate_attachment_request + def validate_thumbnail_request(user_profile: UserProfile, path: str) -> Optional[bool]: # path here does not have a leading / as it is parsed from request hitting the diff --git a/zerver/views/tutorial.py b/zerver/views/tutorial.py index 5b647af7f3..93ff426061 100644 --- a/zerver/views/tutorial.py +++ b/zerver/views/tutorial.py @@ -1,11 +1,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import human_users_only -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_string from zerver.models import UserProfile + @human_users_only @has_request_variables def set_tutorial_status(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/views/typing.py b/zerver/views/typing.py index 08f5ca1b01..1b2d143a71 100644 --- a/zerver/views/typing.py +++ b/zerver/views/typing.py @@ -1,12 +1,14 @@ -from django.http import HttpRequest, HttpResponse from typing import List -from zerver.decorator import has_request_variables, REQ +from django.http import HttpRequest, HttpResponse + +from zerver.decorator import REQ, has_request_variables from zerver.lib.actions import check_send_typing_notification from zerver.lib.response import json_success from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile + @has_request_variables def send_notification_backend( request: HttpRequest, diff --git a/zerver/views/unsubscribe.py b/zerver/views/unsubscribe.py index 4b5b561281..d22a2e8190 100644 --- a/zerver/views/unsubscribe.py +++ b/zerver/views/unsubscribe.py @@ -1,13 +1,14 @@ -from django.http import HttpRequest, HttpResponse -from django.shortcuts import render from typing import Callable -from confirmation.models import Confirmation, get_object_from_key, \ - ConfirmationKeyException +from django.http import HttpRequest, HttpResponse +from django.shortcuts import render + +from confirmation.models import Confirmation, ConfirmationKeyException, get_object_from_key +from zerver.context_processors import common_context from zerver.lib.actions import do_change_notification_settings from zerver.lib.send_email import clear_scheduled_emails -from zerver.models import UserProfile, ScheduledEmail -from zerver.context_processors import common_context +from zerver.models import ScheduledEmail, UserProfile + def process_unsubscribe(request: HttpRequest, confirmation_key: str, subscription_type: str, unsubscribe_function: Callable[[UserProfile], None]) -> HttpResponse: diff --git a/zerver/views/upload.py b/zerver/views/upload.py index 22ff8389b4..eaa724e29b 100644 --- a/zerver/views/upload.py +++ b/zerver/views/upload.py @@ -1,17 +1,24 @@ -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, \ - HttpResponseNotFound +from mimetypes import guess_type + +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound from django.shortcuts import redirect from django.utils.cache import patch_cache_control from django.utils.translation import ugettext as _ - -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_message_image_from_request, get_local_file_path, \ - get_signed_upload_url, check_upload_within_quota, INLINE_MIME_TYPES, \ - generate_unauthed_file_access_url, get_local_file_path_id_from_token -from zerver.models import UserProfile, validate_attachment_request -from django.conf import settings from django_sendfile import sendfile -from mimetypes import guess_type + +from zerver.lib.response import json_error, json_success +from zerver.lib.upload import ( + INLINE_MIME_TYPES, + check_upload_within_quota, + generate_unauthed_file_access_url, + get_local_file_path, + get_local_file_path_id_from_token, + get_signed_upload_url, + upload_message_image_from_request, +) +from zerver.models import UserProfile, validate_attachment_request + def serve_s3(request: HttpRequest, url_path: str, url_only: bool) -> HttpResponse: url = get_signed_upload_url(url_path) diff --git a/zerver/views/user_groups.py b/zerver/views/user_groups.py index 2aaddeda87..852823ee63 100644 --- a/zerver/views/user_groups.py +++ b/zerver/views/user_groups.py @@ -1,21 +1,31 @@ -from django.http import HttpResponse, HttpRequest -from django.utils.translation import ugettext as _ - from typing import List +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + from zerver.decorator import require_member_or_admin, require_user_group_edit_permission -from zerver.lib.actions import check_add_user_group, do_update_user_group_name, \ - do_update_user_group_description, bulk_add_members_to_user_group, \ - remove_members_from_user_group, check_delete_user_group +from zerver.lib.actions import ( + bulk_add_members_to_user_group, + check_add_user_group, + check_delete_user_group, + do_update_user_group_description, + do_update_user_group_name, + remove_members_from_user_group, +) from zerver.lib.exceptions import JsonableError -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.user_groups import ( + access_user_group_by_id, + get_memberships_of_users, + get_user_group_members, + user_groups_in_realm_serialized, +) from zerver.lib.users import user_ids_to_users -from zerver.lib.validator import check_list, check_int -from zerver.lib.user_groups import access_user_group_by_id, get_memberships_of_users, \ - get_user_group_members, user_groups_in_realm_serialized +from zerver.lib.validator import check_int, check_list from zerver.models import UserProfile -from zerver.views.streams import compose_views, FuncKwargPair +from zerver.views.streams import FuncKwargPair, compose_views + @require_user_group_edit_permission @has_request_variables diff --git a/zerver/views/user_settings.py b/zerver/views/user_settings.py index 86999d063a..b19fe08ff3 100644 --- a/zerver/views/user_settings.py +++ b/zerver/views/user_settings.py @@ -1,35 +1,47 @@ -from typing import Optional, Any, Dict +from typing import Any, Dict, Optional -from django.core.exceptions import ValidationError -from django.utils.translation import ugettext as _ from django.conf import settings from django.contrib.auth import authenticate, update_session_auth_hash +from django.core.exceptions import ValidationError from django.http import HttpRequest, HttpResponse from django.shortcuts import render +from django.utils.translation import ugettext as _ -from zerver.decorator import has_request_variables, \ - REQ, human_users_only -from zerver.lib.actions import do_change_password, do_change_notification_settings, \ - do_change_enter_sends, do_regenerate_api_key, do_change_avatar_fields, \ - do_set_user_display_setting, do_change_user_delivery_email, \ - do_start_email_change_process, check_change_full_name, \ - get_available_notification_sounds, validate_email_is_valid +from confirmation.models import ( + Confirmation, + ConfirmationKeyException, + get_object_from_key, + render_confirmation_key_error, +) +from zerver.decorator import REQ, has_request_variables, human_users_only +from zerver.lib.actions import ( + check_change_full_name, + do_change_avatar_fields, + do_change_enter_sends, + do_change_notification_settings, + do_change_password, + do_change_user_delivery_email, + do_regenerate_api_key, + do_set_user_display_setting, + do_start_email_change_process, + get_available_notification_sounds, + validate_email_is_valid, +) from zerver.lib.avatar import avatar_url -from zerver.lib.email_validation import get_realm_email_validator, \ - validate_email_not_already_in_realm -from zerver.lib.send_email import send_email, FromAddress +from zerver.lib.email_validation import ( + get_realm_email_validator, + validate_email_not_already_in_realm, +) from zerver.lib.i18n import get_available_language_codes -from zerver.lib.response import json_success, json_error -from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_int_in, \ - check_string_in from zerver.lib.rate_limiter import RateLimited from zerver.lib.request import JsonableError +from zerver.lib.response import json_error, json_success +from zerver.lib.send_email import FromAddress, send_email from zerver.lib.timezone import get_all_timezones -from zerver.models import UserProfile, name_changes_disabled, avatar_changes_disabled -from confirmation.models import get_object_from_key, render_confirmation_key_error, \ - ConfirmationKeyException, Confirmation -from zproject.backends import email_belongs_to_ldap, check_password_strength +from zerver.lib.upload import upload_avatar_image +from zerver.lib.validator import check_bool, check_int, check_int_in, check_string, check_string_in +from zerver.models import UserProfile, avatar_changes_disabled, name_changes_disabled +from zproject.backends import check_password_strength, email_belongs_to_ldap AVATAR_CHANGES_DISABLED_ERROR = _("Avatar changes are disabled in this organization.") diff --git a/zerver/views/users.py b/zerver/views/users.py index 6e8af2d747..fd5e926f3a 100644 --- a/zerver/views/users.py +++ b/zerver/views/users.py @@ -1,44 +1,84 @@ -from typing import Union, Optional, Dict, Any, List +from typing import Any, Dict, List, Optional, Union -from django.http import HttpRequest, HttpResponse - -from django.utils.translation import ugettext as _ -from django.shortcuts import redirect from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect +from django.utils.translation import ugettext as _ -from zerver.decorator import require_realm_admin, require_member_or_admin -from zerver.forms import CreateUserForm, PASSWORD_TOO_WEAK_ERROR -from zerver.lib.actions import do_change_avatar_fields, do_change_bot_owner, \ - do_change_user_role, do_change_default_all_public_streams, \ - do_change_default_events_register_stream, do_change_default_sending_stream, \ - do_create_user, do_deactivate_user, do_reactivate_user, do_regenerate_api_key, \ - check_change_full_name, notify_created_bot, do_update_outgoing_webhook_service, \ - do_update_bot_config_data, check_change_bot_full_name, \ - do_update_user_custom_profile_data_if_changed, check_remove_custom_profile_field_value +from zerver.decorator import require_member_or_admin, require_realm_admin +from zerver.forms import PASSWORD_TOO_WEAK_ERROR, CreateUserForm +from zerver.lib.actions import ( + check_change_bot_full_name, + check_change_full_name, + check_remove_custom_profile_field_value, + do_change_avatar_fields, + do_change_bot_owner, + do_change_default_all_public_streams, + do_change_default_events_register_stream, + do_change_default_sending_stream, + do_change_user_role, + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_regenerate_api_key, + do_update_bot_config_data, + do_update_outgoing_webhook_service, + do_update_user_custom_profile_data_if_changed, + notify_created_bot, +) from zerver.lib.avatar import avatar_url, get_gravatar_url from zerver.lib.bot_config import set_bot_config from zerver.lib.email_validation import email_allowed_for_realm from zerver.lib.exceptions import CannotDeactivateLastUserError from zerver.lib.integrations import EMBEDDED_BOTS -from zerver.lib.request import has_request_variables, REQ +from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.streams import access_stream_by_name, access_stream_by_id, subscribed_to_stream +from zerver.lib.streams import access_stream_by_id, access_stream_by_name, subscribed_to_stream from zerver.lib.upload import upload_avatar_image -from zerver.lib.validator import check_bool, check_string, check_int, check_url, check_dict, check_list, \ - check_int_in from zerver.lib.url_encoding import add_query_arg_to_redirect_url -from zerver.lib.users import check_valid_bot_type, check_bot_creation_policy, \ - check_full_name, check_short_name, check_valid_interface_type, check_valid_bot_config, \ - access_bot_by_id, add_service, access_user_by_id, check_bot_name_available, \ - validate_user_custom_profile_data, get_raw_user_data, get_api_key +from zerver.lib.users import ( + access_bot_by_id, + access_user_by_id, + add_service, + check_bot_creation_policy, + check_bot_name_available, + check_full_name, + check_short_name, + check_valid_bot_config, + check_valid_bot_type, + check_valid_interface_type, + get_api_key, + get_raw_user_data, + validate_user_custom_profile_data, +) from zerver.lib.utils import generate_api_key -from zerver.models import UserProfile, Stream, Message, \ - get_user_by_delivery_email, Service, get_user_including_cross_realm, \ - DomainNotAllowedForRealmError, DisposableEmailError, get_user_profile_by_id_in_realm, \ - EmailContainsPlusError, get_user_by_id_in_realm_including_cross_realm, Realm, \ - InvalidFakeEmailDomain +from zerver.lib.validator import ( + check_bool, + check_dict, + check_int, + check_int_in, + check_list, + check_string, + check_url, +) +from zerver.models import ( + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + InvalidFakeEmailDomain, + Message, + Realm, + Service, + Stream, + UserProfile, + get_user_by_delivery_email, + get_user_by_id_in_realm_including_cross_realm, + get_user_including_cross_realm, + get_user_profile_by_id_in_realm, +) from zproject.backends import check_password_strength + def check_last_owner(user_profile: UserProfile) -> bool: owners = set(user_profile.realm.get_human_owner_users()) return user_profile.is_realm_owner and not user_profile.is_bot and len(owners) == 1 diff --git a/zerver/views/video_calls.py b/zerver/views/video_calls.py index a1c43f084b..f36373702f 100644 --- a/zerver/views/video_calls.py +++ b/zerver/views/video_calls.py @@ -1,10 +1,11 @@ -from functools import partial import json +from functools import partial from typing import Dict from urllib.parse import urljoin +import requests from django.conf import settings -from django.http import HttpResponse, HttpRequest +from django.http import HttpRequest, HttpResponse from django.middleware import csrf from django.shortcuts import redirect, render from django.utils.crypto import constant_time_compare, salted_hmac @@ -13,7 +14,6 @@ from django.views.decorators.cache import never_cache from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST from oauthlib.oauth2 import OAuth2Error -import requests from requests_oauthlib import OAuth2Session from zerver.decorator import REQ, has_request_variables, zulip_login_required diff --git a/zerver/views/zephyr.py b/zerver/views/zephyr.py index 99db0796ae..71c12ce588 100644 --- a/zerver/views/zephyr.py +++ b/zerver/views/zephyr.py @@ -1,22 +1,21 @@ -from django.conf import settings -from django.http import HttpResponse, HttpRequest -from django.utils.translation import ugettext as _ -from zerver.decorator import authenticated_json_view -from zerver.lib.ccache import make_ccache -from zerver.lib.pysa import mark_sanitized -from zerver.lib.request import has_request_variables, REQ -from zerver.lib.response import json_success, json_error -from zerver.lib.users import get_api_key -from zerver.models import UserProfile - import base64 import logging import re import subprocess -import ujson - from typing import Optional +import ujson +from django.conf import settings +from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ + +from zerver.decorator import authenticated_json_view +from zerver.lib.ccache import make_ccache +from zerver.lib.pysa import mark_sanitized +from zerver.lib.request import REQ, has_request_variables +from zerver.lib.response import json_error, json_success +from zerver.lib.users import get_api_key +from zerver.models import UserProfile # Hack for mit.edu users whose Kerberos usernames don't match what they zephyr # as. The key is for Kerberos and the value is for zephyr. diff --git a/zerver/webhooks/alertmanager/view.py b/zerver/webhooks/alertmanager/view.py index 4156da5e72..11177d1008 100644 --- a/zerver/webhooks/alertmanager/view.py +++ b/zerver/webhooks/alertmanager/view.py @@ -9,6 +9,7 @@ from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile + @api_key_only_webhook_view('AlertManager') @has_request_variables def api_alertmanager_webhook(request: HttpRequest, user_profile: UserProfile, diff --git a/zerver/webhooks/ansibletower/view.py b/zerver/webhooks/ansibletower/view.py index b41329c8f5..77d0676d18 100644 --- a/zerver/webhooks/ansibletower/view.py +++ b/zerver/webhooks/ansibletower/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict, List from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/appveyor/view.py b/zerver/webhooks/appveyor/view.py index 9caadc9fe8..40914e1ef2 100644 --- a/zerver/webhooks/appveyor/view.py +++ b/zerver/webhooks/appveyor/view.py @@ -2,8 +2,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/basecamp/view.py b/zerver/webhooks/basecamp/view.py index 8ba5412070..74793d3826 100644 --- a/zerver/webhooks/basecamp/view.py +++ b/zerver/webhooks/basecamp/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .support_event import SUPPORT_EVENTS diff --git a/zerver/webhooks/beanstalk/tests.py b/zerver/webhooks/beanstalk/tests.py index 162ce7b708..fe6f56e022 100644 --- a/zerver/webhooks/beanstalk/tests.py +++ b/zerver/webhooks/beanstalk/tests.py @@ -1,5 +1,4 @@ from typing import Dict - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/beanstalk/view.py b/zerver/webhooks/beanstalk/view.py index 114642cdc3..2e23e45788 100644 --- a/zerver/webhooks/beanstalk/view.py +++ b/zerver/webhooks/beanstalk/view.py @@ -12,8 +12,7 @@ from zerver.lib.response import json_success from zerver.lib.types import ViewFuncT from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket/tests.py b/zerver/webhooks/bitbucket/tests.py index b1979283ad..61fe6cf6ac 100644 --- a/zerver/webhooks/bitbucket/tests.py +++ b/zerver/webhooks/bitbucket/tests.py @@ -1,5 +1,4 @@ from typing import Dict, Union - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/bitbucket/view.py b/zerver/webhooks/bitbucket/view.py index 583ef1ce71..e2d9579add 100644 --- a/zerver/webhooks/bitbucket/view.py +++ b/zerver/webhooks/bitbucket/view.py @@ -7,8 +7,7 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_dict from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - get_push_commits_event_message +from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, get_push_commits_event_message from zerver.models import UserProfile diff --git a/zerver/webhooks/bitbucket2/view.py b/zerver/webhooks/bitbucket2/view.py index 1333b62f77..35bae5d0ba 100644 --- a/zerver/webhooks/bitbucket2/view.py +++ b/zerver/webhooks/bitbucket2/view.py @@ -10,13 +10,22 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_force_push_commits_event_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_force_push_commits_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile BITBUCKET_TOPIC_TEMPLATE = '{repository_name}' diff --git a/zerver/webhooks/bitbucket3/view.py b/zerver/webhooks/bitbucket3/view.py index 29dc805f03..cb6bae92d7 100644 --- a/zerver/webhooks/bitbucket3/view.py +++ b/zerver/webhooks/bitbucket3/view.py @@ -8,16 +8,23 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_create_branch_event_message, \ - get_pull_request_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_create_branch_event_message, + get_pull_request_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile -from zerver.webhooks.bitbucket2.view import BITBUCKET_FORK_BODY, \ - BITBUCKET_REPO_UPDATED_CHANGED, BITBUCKET_TOPIC_TEMPLATE +from zerver.webhooks.bitbucket2.view import ( + BITBUCKET_FORK_BODY, + BITBUCKET_REPO_UPDATED_CHANGED, + BITBUCKET_TOPIC_TEMPLATE, +) BRANCH_UPDATED_MESSAGE_TEMPLATE = "{user_name} pushed to branch {branch_name}. Head is now {head}." PULL_REQUEST_MARKED_AS_NEEDS_WORK_TEMPLATE = "{user_name} marked [PR #{number}]({url}) as \"needs work\"." diff --git a/zerver/webhooks/clubhouse/tests.py b/zerver/webhooks/clubhouse/tests.py index 72f802c430..76494504ea 100644 --- a/zerver/webhooks/clubhouse/tests.py +++ b/zerver/webhooks/clubhouse/tests.py @@ -1,5 +1,4 @@ import json - from unittest.mock import MagicMock, patch from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/clubhouse/view.py b/zerver/webhooks/clubhouse/view.py index 68628753e9..2987cddd72 100644 --- a/zerver/webhooks/clubhouse/view.py +++ b/zerver/webhooks/clubhouse/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile EPIC_NAME_TEMPLATE = "**{name}**" diff --git a/zerver/webhooks/flock/view.py b/zerver/webhooks/flock/view.py index 99f2232303..dd8938387d 100644 --- a/zerver/webhooks/flock/view.py +++ b/zerver/webhooks/flock/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/github/view.py b/zerver/webhooks/github/view.py index 7fd1510f04..1e677e0c67 100644 --- a/zerver/webhooks/github/view.py +++ b/zerver/webhooks/github/view.py @@ -8,15 +8,24 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import CONTENT_MESSAGE_TEMPLATE, \ - TOPIC_WITH_BRANCH_TEMPLATE, TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, \ - get_commits_comment_action_message, get_issue_event_message, \ - get_pull_request_event_message, get_push_commits_event_message, \ - get_push_tag_event_message, get_setup_webhook_message, \ - get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + CONTENT_MESSAGE_TEMPLATE, + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_release_event_message, + get_setup_webhook_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GITHUB_EVENT") diff --git a/zerver/webhooks/gitlab/view.py b/zerver/webhooks/gitlab/view.py index 80a6178070..6e10e2e19c 100644 --- a/zerver/webhooks/gitlab/view.py +++ b/zerver/webhooks/gitlab/view.py @@ -8,13 +8,21 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, validate_extract_webhook_http_header -from zerver.lib.webhooks.git import EMPTY_SHA, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, get_commits_comment_action_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_push_tag_event_message, \ - get_remove_branch_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + EMPTY_SHA, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + get_commits_comment_action_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_push_tag_event_message, + get_remove_branch_event_message, +) from zerver.models import UserProfile diff --git a/zerver/webhooks/gogs/view.py b/zerver/webhooks/gogs/view.py index e43f2b75ad..2f4a46300f 100644 --- a/zerver/webhooks/gogs/view.py +++ b/zerver/webhooks/gogs/view.py @@ -6,14 +6,22 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header -from zerver.lib.webhooks.git import TOPIC_WITH_BRANCH_TEMPLATE, \ - TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, TOPIC_WITH_RELEASE_TEMPLATE, \ - get_create_branch_event_message, \ - get_issue_event_message, get_pull_request_event_message, \ - get_push_commits_event_message, get_release_event_message +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) +from zerver.lib.webhooks.git import ( + TOPIC_WITH_BRANCH_TEMPLATE, + TOPIC_WITH_PR_OR_ISSUE_INFO_TEMPLATE, + TOPIC_WITH_RELEASE_TEMPLATE, + get_create_branch_event_message, + get_issue_event_message, + get_pull_request_event_message, + get_push_commits_event_message, + get_release_event_message, +) from zerver.models import UserProfile fixture_to_headers = get_http_headers_from_filename("HTTP_X_GOGS_EVENT") diff --git a/zerver/webhooks/gosquared/view.py b/zerver/webhooks/gosquared/view.py index ec6aacd5a3..0c9affba7d 100644 --- a/zerver/webhooks/gosquared/view.py +++ b/zerver/webhooks/gosquared/view.py @@ -5,8 +5,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TRAFFIC_SPIKE_TEMPLATE = '[{website_name}]({website_url}) has {user_num} visitors online.' diff --git a/zerver/webhooks/grafana/view.py b/zerver/webhooks/grafana/view.py index 9051d7aaec..ebd70a8b66 100644 --- a/zerver/webhooks/grafana/view.py +++ b/zerver/webhooks/grafana/view.py @@ -3,9 +3,9 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.webhooks.common import check_send_webhook_message from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success +from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile GRAFANA_TOPIC_TEMPLATE = '{alert_title}' diff --git a/zerver/webhooks/groove/view.py b/zerver/webhooks/groove/view.py index 76b508056f..038003bdfa 100644 --- a/zerver/webhooks/groove/view.py +++ b/zerver/webhooks/groove/view.py @@ -7,9 +7,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile TICKET_STARTED_TEMPLATE = """ diff --git a/zerver/webhooks/harbor/view.py b/zerver/webhooks/harbor/view.py index 64a6846cb7..3dd7a35c05 100644 --- a/zerver/webhooks/harbor/view.py +++ b/zerver/webhooks/harbor/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile IGNORED_EVENTS = [ diff --git a/zerver/webhooks/hellosign/tests.py b/zerver/webhooks/hellosign/tests.py index 2cadf71664..79738fddac 100644 --- a/zerver/webhooks/hellosign/tests.py +++ b/zerver/webhooks/hellosign/tests.py @@ -1,6 +1,8 @@ -from zerver.lib.test_classes import WebhookTestCase from typing import Dict +from zerver.lib.test_classes import WebhookTestCase + + class HelloSignHookTests(WebhookTestCase): STREAM_NAME = 'hellosign' URL_TEMPLATE = "/api/v1/external/hellosign?stream={stream}&api_key={api_key}" diff --git a/zerver/webhooks/hellosign/view.py b/zerver/webhooks/hellosign/view.py index 6dd775af05..27018dd6b3 100644 --- a/zerver/webhooks/hellosign/view.py +++ b/zerver/webhooks/hellosign/view.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List +import ujson from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view @@ -7,7 +8,6 @@ from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile -import ujson IS_AWAITING_SIGNATURE = "is awaiting the signature of {awaiting_recipients}" WAS_JUST_SIGNED_BY = "was just signed by {signed_recipients}" diff --git a/zerver/webhooks/insping/view.py b/zerver/webhooks/insping/view.py index 152de3ad44..3acea44451 100644 --- a/zerver/webhooks/insping/view.py +++ b/zerver/webhooks/insping/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/intercom/view.py b/zerver/webhooks/intercom/view.py index 35960af125..fcee857de7 100644 --- a/zerver/webhooks/intercom/view.py +++ b/zerver/webhooks/intercom/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile COMPANY_CREATED = """ diff --git a/zerver/webhooks/jira/view.py b/zerver/webhooks/jira/view.py index 05f8c6aef7..ab095687df 100644 --- a/zerver/webhooks/jira/view.py +++ b/zerver/webhooks/jira/view.py @@ -9,8 +9,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import Realm, UserProfile, get_user_by_delivery_email IGNORED_EVENTS = [ diff --git a/zerver/webhooks/netlify/view.py b/zerver/webhooks/netlify/view.py index 1f06303fca..1c256b1dfe 100644 --- a/zerver/webhooks/netlify/view.py +++ b/zerver/webhooks/netlify/view.py @@ -5,9 +5,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile EVENTS = ['deploy_failed', 'deploy_locked', 'deploy_unlocked', 'deploy_building', 'deploy_created'] diff --git a/zerver/webhooks/newrelic/view.py b/zerver/webhooks/newrelic/view.py index 7848c811ab..8ef9bf0f2e 100644 --- a/zerver/webhooks/newrelic/view.py +++ b/zerver/webhooks/newrelic/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_dict -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile ALERT_TEMPLATE = "{long_description} ([view alert]({alert_url}))." diff --git a/zerver/webhooks/pagerduty/view.py b/zerver/webhooks/pagerduty/view.py index 69576c38df..9267bb9811 100644 --- a/zerver/webhooks/pagerduty/view.py +++ b/zerver/webhooks/pagerduty/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PAGER_DUTY_EVENT_NAMES = { diff --git a/zerver/webhooks/pingdom/view.py b/zerver/webhooks/pingdom/view.py index 1cb1ec5aed..16ac895043 100644 --- a/zerver/webhooks/pingdom/view.py +++ b/zerver/webhooks/pingdom/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile PINGDOM_TOPIC_TEMPLATE = '{name} status.' diff --git a/zerver/webhooks/pivotal/view.py b/zerver/webhooks/pivotal/view.py index 3a56f4b2dd..6fd45a8275 100644 --- a/zerver/webhooks/pivotal/view.py +++ b/zerver/webhooks/pivotal/view.py @@ -10,8 +10,7 @@ from django.utils.translation import ugettext as _ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/raygun/view.py b/zerver/webhooks/raygun/view.py index cace93e5bb..497c9c9e27 100644 --- a/zerver/webhooks/raygun/view.py +++ b/zerver/webhooks/raygun/view.py @@ -6,8 +6,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/reviewboard/view.py b/zerver/webhooks/reviewboard/view.py index e89d2122b1..e5e024d3ef 100644 --- a/zerver/webhooks/reviewboard/view.py +++ b/zerver/webhooks/reviewboard/view.py @@ -5,9 +5,12 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message, get_http_headers_from_filename, \ - validate_extract_webhook_http_header +from zerver.lib.webhooks.common import ( + UnexpectedWebhookEventType, + check_send_webhook_message, + get_http_headers_from_filename, + validate_extract_webhook_http_header, +) from zerver.models import UserProfile REVIEW_REQUEST_PUBLISHED = """ diff --git a/zerver/webhooks/semaphore/tests.py b/zerver/webhooks/semaphore/tests.py index bcb487f17a..83356b5601 100644 --- a/zerver/webhooks/semaphore/tests.py +++ b/zerver/webhooks/semaphore/tests.py @@ -1,5 +1,7 @@ -import ujson from unittest.mock import patch + +import ujson + from zerver.lib.test_classes import WebhookTestCase diff --git a/zerver/webhooks/semaphore/view.py b/zerver/webhooks/semaphore/view.py index 77b0979a86..4fe0fc3242 100644 --- a/zerver/webhooks/semaphore/view.py +++ b/zerver/webhooks/semaphore/view.py @@ -1,6 +1,5 @@ # Webhooks for external integrations. -from typing import Any, Dict, Tuple, Optional - +from typing import Any, Dict, Optional, Tuple from urllib.parse import urlparse from django.http import HttpRequest, HttpResponse diff --git a/zerver/webhooks/sentry/view.py b/zerver/webhooks/sentry/view.py index 9d2aa7977c..4e42b6c31c 100644 --- a/zerver/webhooks/sentry/view.py +++ b/zerver/webhooks/sentry/view.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Optional, Tuple from django.http import HttpRequest, HttpResponse @@ -8,7 +8,6 @@ from zerver.lib.response import json_success from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile - DEPRECATED_EXCEPTION_MESSAGE_TEMPLATE = """ New [issue]({url}) (level: {level}): diff --git a/zerver/webhooks/slack_incoming/view.py b/zerver/webhooks/slack_incoming/view.py index 418953c657..67eb9d9003 100644 --- a/zerver/webhooks/slack_incoming/view.py +++ b/zerver/webhooks/slack_incoming/view.py @@ -1,17 +1,18 @@ # Webhooks for external integrations. +import re from typing import Any, Dict, Optional +import ujson from django.http import HttpRequest, HttpResponse +from django.utils.translation import ugettext as _ from zerver.decorator import api_key_only_webhook_view +from zerver.lib.exceptions import InvalidJSONError from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message -from zerver.lib.exceptions import InvalidJSONError -from django.utils.translation import ugettext as _ from zerver.models import UserProfile -import re -import ujson + @api_key_only_webhook_view('SlackIncoming') @has_request_variables diff --git a/zerver/webhooks/statuspage/view.py b/zerver/webhooks/statuspage/view.py index e5fa6b3f3b..cf8e820b31 100644 --- a/zerver/webhooks/statuspage/view.py +++ b/zerver/webhooks/statuspage/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.response import json_success from zerver.lib.webhooks.common import check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/stripe/view.py b/zerver/webhooks/stripe/view.py index 57d542f748..4b2ed311b9 100644 --- a/zerver/webhooks/stripe/view.py +++ b/zerver/webhooks/stripe/view.py @@ -8,8 +8,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/teamcity/tests.py b/zerver/webhooks/teamcity/tests.py index 268c90d078..db3e8acee7 100644 --- a/zerver/webhooks/teamcity/tests.py +++ b/zerver/webhooks/teamcity/tests.py @@ -3,8 +3,7 @@ import ujson from zerver.lib.send_email import FromAddress from zerver.lib.test_classes import WebhookTestCase from zerver.models import Recipient, get_realm, get_user_by_delivery_email -from zerver.webhooks.teamcity.view import \ - MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE +from zerver.webhooks.teamcity.view import MISCONFIGURED_PAYLOAD_TYPE_ERROR_MESSAGE class TeamcityHookTests(WebhookTestCase): diff --git a/zerver/webhooks/teamcity/view.py b/zerver/webhooks/teamcity/view.py index 2dc4dcecf7..b83cfb0c82 100644 --- a/zerver/webhooks/teamcity/view.py +++ b/zerver/webhooks/teamcity/view.py @@ -6,8 +6,10 @@ from django.db.models import Q from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view -from zerver.lib.actions import check_send_private_message, \ - send_rate_limited_pm_notification_to_bot_owner +from zerver.lib.actions import ( + check_send_private_message, + send_rate_limited_pm_notification_to_bot_owner, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/webhooks/transifex/view.py b/zerver/webhooks/transifex/view.py index 8614807fa0..7f22564963 100644 --- a/zerver/webhooks/transifex/view.py +++ b/zerver/webhooks/transifex/view.py @@ -7,8 +7,7 @@ from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success from zerver.lib.validator import check_int -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile diff --git a/zerver/webhooks/trello/view/__init__.py b/zerver/webhooks/trello/view/__init__.py index c455383037..e6a01d0c3c 100644 --- a/zerver/webhooks/trello/view/__init__.py +++ b/zerver/webhooks/trello/view/__init__.py @@ -4,17 +4,14 @@ from typing import Any, Mapping, Optional, Tuple import ujson from django.http import HttpRequest, HttpResponse -from zerver.decorator import api_key_only_webhook_view, \ - return_success_on_head_request +from zerver.decorator import api_key_only_webhook_view, return_success_on_head_request from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile from .board_actions import SUPPORTED_BOARD_ACTIONS, process_board_action -from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, \ - process_card_action +from .card_actions import IGNORED_CARD_ACTIONS, SUPPORTED_CARD_ACTIONS, process_card_action @api_key_only_webhook_view('Trello') diff --git a/zerver/webhooks/updown/view.py b/zerver/webhooks/updown/view.py index 8151518822..d0c49b2eca 100644 --- a/zerver/webhooks/updown/view.py +++ b/zerver/webhooks/updown/view.py @@ -7,8 +7,7 @@ from django.http import HttpRequest, HttpResponse from zerver.decorator import api_key_only_webhook_view from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_success -from zerver.lib.webhooks.common import UnexpectedWebhookEventType, \ - check_send_webhook_message +from zerver.lib.webhooks.common import UnexpectedWebhookEventType, check_send_webhook_message from zerver.models import UserProfile TOPIC_TEMPLATE = "{service_url}" diff --git a/zerver/webhooks/zabbix/view.py b/zerver/webhooks/zabbix/view.py index 846dcc4bbd..5a59bd7c82 100644 --- a/zerver/webhooks/zabbix/view.py +++ b/zerver/webhooks/zabbix/view.py @@ -3,8 +3,7 @@ from typing import Any, Dict from django.http import HttpRequest, HttpResponse from django.utils.translation import ugettext as _ -from zerver.decorator import REQ, api_key_only_webhook_view, \ - has_request_variables +from zerver.decorator import REQ, api_key_only_webhook_view, has_request_variables from zerver.lib.actions import send_rate_limited_pm_notification_to_bot_owner from zerver.lib.response import json_error, json_success from zerver.lib.send_email import FromAddress diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 1aa7cb202f..b647706469 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -1,61 +1,97 @@ # Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Mapping, MutableSequence, Optional, cast, Tuple, TypeVar, Type - import copy +import datetime +import email +import logging +import os import signal -import tempfile -from functools import wraps -from threading import Timer - import smtplib import socket +import tempfile +import time +import urllib +from abc import ABC, abstractmethod +from collections import defaultdict, deque +from functools import wraps +from threading import Timer +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + MutableSequence, + Optional, + Tuple, + Type, + TypeVar, + cast, +) +import requests +import ujson from django.conf import settings from django.db import connection from django.utils.timezone import now as timezone_now -from zerver.models import \ - get_client, get_system_bot, PreregistrationUser, \ - get_user_profile_by_id, Message, Realm, UserMessage, UserProfile, \ - Client, flush_per_request_caches -from zerver.lib.context_managers import lockfile -from zerver.lib.error_notify import do_report_error -from zerver.lib.queue import SimpleQueueClient, retry_event -from zerver.lib.timestamp import timestamp_to_datetime -from zerver.lib.email_notifications import handle_missedmessage_emails -from zerver.lib.push_notifications import handle_push_notification, handle_remove_push_notification, \ - initialize_push_notifications, clear_push_device_tokens -from zerver.lib.actions import do_send_confirmation_email, \ - do_update_user_activity, do_update_user_activity_interval, do_update_user_presence, \ - internal_send_private_message, notify_realm_export, \ - render_incoming_message, do_update_embedded_data, do_mark_stream_messages_as_read -from zerver.lib.url_preview import preview as url_preview -from zerver.lib.digest import handle_digest_email -from zerver.lib.send_email import send_future_email, send_email_from_dict, \ - FromAddress, EmailNotDeliveredException, handle_send_email_format_changes -from zerver.lib.email_mirror import process_message as mirror_email, rate_limit_mirror_by_realm, \ - is_missed_message_address, decode_stream_email_address -from zerver.lib.streams import access_stream_by_id -from zerver.lib.db import reset_queries -from zerver.context_processors import common_context -from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler -from zerver.models import get_bot_services, RealmAuditLog from zulip_bots.lib import ExternalBotHandler, extract_query_without_mention -from zerver.lib.bot_lib import EmbeddedBotHandler, get_bot_handler, EmbeddedBotQuitException + +from zerver.context_processors import common_context +from zerver.lib.actions import ( + do_mark_stream_messages_as_read, + do_send_confirmation_email, + do_update_embedded_data, + do_update_user_activity, + do_update_user_activity_interval, + do_update_user_presence, + internal_send_private_message, + notify_realm_export, + render_incoming_message, +) +from zerver.lib.bot_lib import EmbeddedBotHandler, EmbeddedBotQuitException, get_bot_handler +from zerver.lib.context_managers import lockfile +from zerver.lib.db import reset_queries +from zerver.lib.digest import handle_digest_email +from zerver.lib.email_mirror import decode_stream_email_address, is_missed_message_address +from zerver.lib.email_mirror import process_message as mirror_email +from zerver.lib.email_mirror import rate_limit_mirror_by_realm +from zerver.lib.email_notifications import handle_missedmessage_emails +from zerver.lib.error_notify import do_report_error from zerver.lib.exceptions import RateLimited from zerver.lib.export import export_realm_wrapper -from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.outgoing_webhook import do_rest_call, get_outgoing_webhook_service_handler +from zerver.lib.push_notifications import ( + clear_push_device_tokens, + handle_push_notification, + handle_remove_push_notification, + initialize_push_notifications, +) from zerver.lib.pysa import mark_sanitized - -import os -import ujson -from collections import defaultdict, deque -import email -import time -import datetime -import logging -import requests -import urllib +from zerver.lib.queue import SimpleQueueClient, retry_event +from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError +from zerver.lib.send_email import ( + EmailNotDeliveredException, + FromAddress, + handle_send_email_format_changes, + send_email_from_dict, + send_future_email, +) +from zerver.lib.streams import access_stream_by_id +from zerver.lib.timestamp import timestamp_to_datetime +from zerver.lib.url_preview import preview as url_preview +from zerver.models import ( + Client, + Message, + PreregistrationUser, + Realm, + RealmAuditLog, + UserMessage, + UserProfile, + flush_per_request_caches, + get_bot_services, + get_client, + get_system_bot, + get_user_profile_by_id, +) logger = logging.getLogger(__name__) diff --git a/zilencer/forms.py b/zilencer/forms.py index 4a2352e749..b6f90c3832 100644 --- a/zilencer/forms.py +++ b/zilencer/forms.py @@ -1,5 +1,6 @@ from django import forms + class EnterpriseToSForm(forms.Form): full_name = forms.CharField(max_length=100) company = forms.CharField(max_length=100) diff --git a/zilencer/management/commands/add_mock_conversation.py b/zilencer/management/commands/add_mock_conversation.py index 69f40877e2..ae50776598 100644 --- a/zilencer/management/commands/add_mock_conversation.py +++ b/zilencer/management/commands/add_mock_conversation.py @@ -2,9 +2,15 @@ from typing import Any, Dict, List from django.core.management.base import BaseCommand -from zerver.lib.actions import bulk_add_subscriptions, do_add_reaction, \ - do_change_avatar_fields, do_create_user, do_send_messages, ensure_stream, \ - internal_prep_stream_message +from zerver.lib.actions import ( + bulk_add_subscriptions, + do_add_reaction, + do_change_avatar_fields, + do_create_user, + do_send_messages, + ensure_stream, + internal_prep_stream_message, +) from zerver.lib.emoji import emoji_name_to_emoji_code from zerver.lib.upload import upload_avatar_image from zerver.models import Message, UserProfile, get_realm diff --git a/zilencer/management/commands/add_new_realm.py b/zilencer/management/commands/add_new_realm.py index a78723114d..170952ce56 100644 --- a/zilencer/management/commands/add_new_realm.py +++ b/zilencer/management/commands/add_new_realm.py @@ -1,7 +1,6 @@ from typing import Any -from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, \ - do_create_user +from zerver.lib.actions import bulk_add_subscriptions, do_create_realm, do_create_user from zerver.lib.management import ZulipBaseCommand from zerver.lib.onboarding import send_initial_realm_messages from zerver.models import Realm, UserProfile diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index 86c3421ad8..0676eaa773 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -1,11 +1,11 @@ import itertools import os import random -from typing import Any, Callable, Dict, List, \ - Mapping, Sequence, Tuple - -import ujson from datetime import datetime +from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple + +import pylibmc +import ujson from django.conf import settings from django.contrib.sessions.models import Session from django.core.management import call_command @@ -13,11 +13,17 @@ from django.core.management.base import BaseCommand, CommandParser from django.db.models import F, Max from django.utils.timezone import now as timezone_now from django.utils.timezone import timedelta as timezone_timedelta -import pylibmc -from zerver.lib.actions import STREAM_ASSIGNMENT_COLORS, check_add_realm_emoji, \ - do_change_user_role, do_send_messages, do_update_user_custom_profile_data_if_changed, \ - try_add_realm_custom_profile_field, try_add_realm_default_custom_profile_field +from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.lib.actions import ( + STREAM_ASSIGNMENT_COLORS, + check_add_realm_emoji, + do_change_user_role, + do_send_messages, + do_update_user_custom_profile_data_if_changed, + try_add_realm_custom_profile_field, + try_add_realm_default_custom_profile_field, +) from zerver.lib.bulk_create import bulk_create_streams from zerver.lib.cache import cache_set from zerver.lib.generate_test_data import create_test_data, generate_topics @@ -25,18 +31,35 @@ from zerver.lib.onboarding import create_if_missing_realm_internal_bots from zerver.lib.push_notifications import logger as push_notifications_logger from zerver.lib.server_initialization import create_internal_realm, create_users from zerver.lib.storage import static_path -from zerver.lib.users import add_service +from zerver.lib.types import ProfileFieldData from zerver.lib.url_preview.preview import CACHE_NAME as PREVIEW_CACHE_NAME from zerver.lib.user_groups import create_user_group +from zerver.lib.users import add_service from zerver.lib.utils import generate_api_key -from zerver.models import CustomProfileField, DefaultStream, Message, Realm, RealmAuditLog, \ - RealmDomain, Recipient, Service, Stream, Subscription, \ - UserMessage, UserPresence, UserProfile, Huddle, Client, \ - get_client, get_huddle, get_realm, get_stream, \ - get_user, get_user_by_delivery_email, get_user_profile_by_id -from zerver.lib.types import ProfileFieldData - -from scripts.lib.zulip_tools import get_or_create_dev_uuid_var_path +from zerver.models import ( + Client, + CustomProfileField, + DefaultStream, + Huddle, + Message, + Realm, + RealmAuditLog, + RealmDomain, + Recipient, + Service, + Stream, + Subscription, + UserMessage, + UserPresence, + UserProfile, + get_client, + get_huddle, + get_realm, + get_stream, + get_user, + get_user_by_delivery_email, + get_user_profile_by_id, +) settings.TORNADO_SERVER = None # Disable using memcached caches to avoid 'unsupported pickle diff --git a/zilencer/models.py b/zilencer/models.py index 91f93606cb..34481bbeb5 100644 --- a/zilencer/models.py +++ b/zilencer/models.py @@ -2,8 +2,9 @@ import datetime from django.db import models -from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog from analytics.models import BaseCount +from zerver.models import AbstractPushDeviceToken, AbstractRealmAuditLog + def get_remote_server_by_uuid(uuid: str) -> 'RemoteZulipServer': return RemoteZulipServer.objects.get(uuid=uuid) diff --git a/zilencer/views.py b/zilencer/views.py index fc137a4149..ad659c4f1c 100644 --- a/zilencer/views.py +++ b/zilencer/views.py @@ -1,29 +1,46 @@ -from typing import Any, Dict, List, Optional, Union import datetime import logging +from typing import Any, Dict, List, Optional, Union from django.core.exceptions import ValidationError -from django.core.validators import validate_email, URLValidator +from django.core.validators import URLValidator, validate_email from django.db import IntegrityError, transaction from django.http import HttpRequest, HttpResponse from django.utils import timezone -from django.utils.translation import ugettext as _, ugettext as err_ +from django.utils.translation import ugettext as _ +from django.utils.translation import ugettext as err_ from django.views.decorators.csrf import csrf_exempt from analytics.lib.counts import COUNT_STATS -from zerver.decorator import require_post, InvalidZulipServerKeyError +from zerver.decorator import InvalidZulipServerKeyError, require_post from zerver.lib.exceptions import JsonableError -from zerver.lib.push_notifications import send_android_push_notification, \ - send_apple_push_notification +from zerver.lib.push_notifications import ( + send_android_push_notification, + send_apple_push_notification, +) from zerver.lib.request import REQ, has_request_variables from zerver.lib.response import json_error, json_success -from zerver.lib.validator import check_int, check_string, \ - check_capped_string, check_string_fixed_length, check_float, check_none_or, \ - check_dict_only, check_list, check_bool +from zerver.lib.validator import ( + check_bool, + check_capped_string, + check_dict_only, + check_float, + check_int, + check_list, + check_none_or, + check_string, + check_string_fixed_length, +) from zerver.models import UserProfile from zerver.views.push_notifications import validate_token -from zilencer.models import RemotePushDeviceToken, RemoteZulipServer, \ - RemoteRealmCount, RemoteInstallationCount, RemoteRealmAuditLog +from zilencer.models import ( + RemoteInstallationCount, + RemotePushDeviceToken, + RemoteRealmAuditLog, + RemoteRealmCount, + RemoteZulipServer, +) + def validate_entity(entity: Union[UserProfile, RemoteZulipServer]) -> RemoteZulipServer: if not isinstance(entity, RemoteZulipServer): diff --git a/zproject/backends.py b/zproject/backends.py index 56e6b11fe4..b53aac21b9 100644 --- a/zproject/backends.py +++ b/zproject/backends.py @@ -15,67 +15,81 @@ import binascii import copy import logging +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, cast + import jwt import magic import ujson -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union, \ - cast -from typing_extensions import TypedDict -from zxcvbn import zxcvbn - -from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, \ - _LDAPUser, ldap_error from decorator import decorator - +from django.conf import settings from django.contrib.auth import authenticate, get_backends from django.contrib.auth.backends import RemoteUserBackend -from django.conf import settings from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.dispatch import receiver, Signal -from django.http import HttpResponse, HttpResponseRedirect, HttpRequest +from django.dispatch import Signal, receiver +from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.urls import reverse from django.utils.translation import ugettext as _ +from django_auth_ldap.backend import LDAPBackend, LDAPReverseEmailSearch, _LDAPUser, ldap_error from jwt.algorithms import RSAAlgorithm from jwt.exceptions import PyJWTError from lxml.etree import XMLSyntaxError -from requests import HTTPError from onelogin.saml2.errors import OneLogin_Saml2_Error from onelogin.saml2.response import OneLogin_Saml2_Response - -from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, \ - GithubTeamOAuth2 -from social_core.backends.azuread import AzureADOAuth2 -from social_core.backends.gitlab import GitLabOAuth2 -from social_core.backends.base import BaseAuth -from social_core.backends.google import GoogleOAuth2 +from requests import HTTPError from social_core.backends.apple import AppleIdAuth +from social_core.backends.azuread import AzureADOAuth2 +from social_core.backends.base import BaseAuth +from social_core.backends.github import GithubOAuth2, GithubOrganizationOAuth2, GithubTeamOAuth2 +from social_core.backends.gitlab import GitLabOAuth2 +from social_core.backends.google import GoogleOAuth2 from social_core.backends.saml import SAMLAuth +from social_core.exceptions import ( + AuthFailed, + AuthMissingParameter, + AuthStateForbidden, + SocialAuthBaseException, +) from social_core.pipeline.partial import partial -from social_core.exceptions import AuthFailed, SocialAuthBaseException, \ - AuthMissingParameter, AuthStateForbidden +from typing_extensions import TypedDict +from zxcvbn import zxcvbn from zerver.decorator import client_is_exempt_from_rate_limiting -from zerver.lib.actions import do_create_user, do_reactivate_user, do_deactivate_user, \ - do_update_user_custom_profile_data_if_changed -from zerver.lib.avatar import is_avatar_new, avatar_url +from zerver.lib.actions import ( + do_create_user, + do_deactivate_user, + do_reactivate_user, + do_update_user_custom_profile_data_if_changed, +) +from zerver.lib.avatar import avatar_url, is_avatar_new from zerver.lib.avatar_hash import user_avatar_content_hash from zerver.lib.create_user import get_role_for_new_user from zerver.lib.dev_ldap_directory import init_fakeldap -from zerver.lib.email_validation import email_allowed_for_realm, \ - validate_email_not_already_in_realm +from zerver.lib.email_validation import email_allowed_for_realm, validate_email_not_already_in_realm from zerver.lib.mobile_auth_otp import is_valid_otp from zerver.lib.rate_limiter import RateLimitedObject +from zerver.lib.redis_utils import get_dict_from_redis, get_redis_client, put_dict_in_redis from zerver.lib.request import JsonableError -from zerver.lib.users import check_full_name, validate_user_custom_profile_field -from zerver.lib.redis_utils import get_redis_client, get_dict_from_redis, put_dict_in_redis from zerver.lib.subdomains import get_subdomain -from zerver.models import CustomProfileField, DisposableEmailError, DomainNotAllowedForRealmError, \ - EmailContainsPlusError, PreregistrationUser, UserProfile, Realm, custom_profile_fields_for_realm, \ - get_user_profile_by_id, remote_user_to_email, \ - email_to_username, get_realm, get_user_by_delivery_email, supported_auth_backends +from zerver.lib.users import check_full_name, validate_user_custom_profile_field +from zerver.models import ( + CustomProfileField, + DisposableEmailError, + DomainNotAllowedForRealmError, + EmailContainsPlusError, + PreregistrationUser, + Realm, + UserProfile, + custom_profile_fields_for_realm, + email_to_username, + get_realm, + get_user_by_delivery_email, + get_user_profile_by_id, + remote_user_to_email, + supported_auth_backends, +) redis_client = get_redis_client() @@ -505,10 +519,11 @@ class ZulipLDAPAuthBackendBase(ZulipAuthMixin, LDAPBackend): def sync_avatar_from_ldap(self, user: UserProfile, ldap_user: _LDAPUser) -> None: if 'avatar' in settings.AUTH_LDAP_USER_ATTR_MAP: # We do local imports here to avoid import loops - from zerver.lib.upload import upload_avatar_image - from zerver.lib.actions import do_change_avatar_fields from io import BytesIO + from zerver.lib.actions import do_change_avatar_fields + from zerver.lib.upload import upload_avatar_image + avatar_attr_name = settings.AUTH_LDAP_USER_ATTR_MAP['avatar'] if avatar_attr_name not in ldap_user.attrs: # nocoverage # If this specific user doesn't have e.g. a @@ -1236,8 +1251,7 @@ def social_auth_finish(backend: Any, comments below as well as login_or_register_remote_user in `zerver/views/auth.py` for the details on how that dispatch works. """ - from zerver.views.auth import (login_or_register_remote_user, - redirect_and_log_into_subdomain) + from zerver.views.auth import login_or_register_remote_user, redirect_and_log_into_subdomain user_profile = kwargs['user_profile'] return_data = kwargs['return_data'] diff --git a/zproject/computed_settings.py b/zproject/computed_settings.py index 7b2be1e10b..f456f6293a 100644 --- a/zproject/computed_settings.py +++ b/zproject/computed_settings.py @@ -1,21 +1,28 @@ -from copy import deepcopy import os -import time import sys +import time +from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import urljoin -from zerver.lib.db import TimeTrackingConnection import zerver.lib.logging_util +from zerver.lib.db import TimeTrackingConnection -from .config import DEPLOY_ROOT, PRODUCTION, DEVELOPMENT, get_secret, get_config, get_from_file_if_exists +from .config import ( + DEPLOY_ROOT, + DEVELOPMENT, + PRODUCTION, + get_config, + get_from_file_if_exists, + get_secret, +) from .configured_settings import ( ADMINS, ALLOWED_HOSTS, - AUTHENTICATION_BACKENDS, AUTH_LDAP_BIND_DN, AUTH_LDAP_CONNECTION_OPTIONS, AUTH_LDAP_SERVER_URI, + AUTHENTICATION_BACKENDS, CAMO_URI, DEBUG, DEBUG_ERROR_REPORTING, @@ -156,6 +163,8 @@ ALLOWED_HOSTS += [EXTERNAL_HOST.split(":")[0], ALLOWED_HOSTS += REALM_HOSTS.values() from django.template.loaders import app_directories + + class TwoFactorLoader(app_directories.Loader): def get_dirs(self) -> List[str]: dirs = super().get_dirs() diff --git a/zproject/config.py b/zproject/config.py index f55c616c1c..1a10b7f1f8 100644 --- a/zproject/config.py +++ b/zproject/config.py @@ -1,6 +1,6 @@ +import configparser import os from typing import Optional, overload -import configparser DEPLOY_ROOT = os.path.realpath(os.path.dirname(os.path.dirname(__file__))) diff --git a/zproject/configured_settings.py b/zproject/configured_settings.py index 10997c8193..fc4f128e83 100644 --- a/zproject/configured_settings.py +++ b/zproject/configured_settings.py @@ -10,6 +10,7 @@ from .default_settings import * # noqa: F401,F403 isort: skip # Import variables like secrets from the prod_settings file # Import prod_settings after determining the deployment/machine type from .config import PRODUCTION + if PRODUCTION: from .prod_settings import * # noqa: F401,F403 isort: skip else: diff --git a/zproject/default_settings.py b/zproject/default_settings.py index 84285801a9..e9cccd61d6 100644 --- a/zproject/default_settings.py +++ b/zproject/default_settings.py @@ -1,10 +1,11 @@ -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional if TYPE_CHECKING: from django_auth_ldap.config import LDAPSearch from typing_extensions import TypedDict -from .config import PRODUCTION, DEVELOPMENT, get_secret +from .config import DEVELOPMENT, PRODUCTION, get_secret + if PRODUCTION: from .prod_settings import EXTERNAL_HOST, ZULIP_ADMINISTRATOR else: diff --git a/zproject/dev_settings.py b/zproject/dev_settings.py index 3cd628a886..c551372800 100644 --- a/zproject/dev_settings.py +++ b/zproject/dev_settings.py @@ -113,6 +113,7 @@ FAKE_LDAP_MODE = None # type: Optional[str] if FAKE_LDAP_MODE: import ldap from django_auth_ldap.config import LDAPSearch + # To understand these parameters, read the docs in # prod_settings_template.py and on ReadTheDocs. LDAP_APPEND_DOMAIN = None diff --git a/zproject/dev_urls.py b/zproject/dev_urls.py index 254b17c981..2b0ea06b62 100644 --- a/zproject/dev_urls.py +++ b/zproject/dev_urls.py @@ -1,16 +1,18 @@ +import os from urllib.parse import urlsplit -from django.conf.urls import url + from django.conf import settings +from django.conf.urls import url from django.conf.urls.static import static from django.contrib.staticfiles.views import serve as staticfiles_serve from django.http import HttpRequest, HttpResponse from django.views.generic import TemplateView -import os from django.views.static import serve -import zerver.views.development.registration + import zerver.views.auth import zerver.views.development.email_log import zerver.views.development.integrations +import zerver.views.development.registration # These URLs are available only in the development environment diff --git a/zproject/email_backends.py b/zproject/email_backends.py index 0215694353..9a082088cc 100644 --- a/zproject/email_backends.py +++ b/zproject/email_backends.py @@ -1,17 +1,16 @@ -import logging - -from typing import List import configparser - +import logging import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +from typing import List from django.conf import settings -from django.core.mail.backends.base import BaseEmailBackend from django.core.mail import EmailMultiAlternatives +from django.core.mail.backends.base import BaseEmailBackend from django.template import loader + def get_forward_address() -> str: config = configparser.ConfigParser() config.read(settings.FORWARD_ADDRESS_CONFIG_FILE) diff --git a/zproject/jinja2/__init__.py b/zproject/jinja2/__init__.py index 138a4ae683..fb8582a2fe 100644 --- a/zproject/jinja2/__init__.py +++ b/zproject/jinja2/__init__.py @@ -2,7 +2,7 @@ from typing import Any from django.conf import settings from django.contrib.staticfiles.storage import staticfiles_storage -from django.template.defaultfilters import slugify, pluralize +from django.template.defaultfilters import pluralize, slugify from django.urls import reverse from django.utils import translation from django.utils.timesince import timesince diff --git a/zproject/legacy_urls.py b/zproject/legacy_urls.py index b7f7a70f36..1366f13be0 100644 --- a/zproject/legacy_urls.py +++ b/zproject/legacy_urls.py @@ -1,9 +1,10 @@ from django.urls import path + import zerver.views -import zerver.views.streams import zerver.views.auth -import zerver.views.tutorial import zerver.views.report +import zerver.views.streams +import zerver.views.tutorial # Future endpoints should add to urls.py, which includes these legacy urls diff --git a/zproject/test_extra_settings.py b/zproject/test_extra_settings.py index 6a6bfd2753..d7d414f275 100644 --- a/zproject/test_extra_settings.py +++ b/zproject/test_extra_settings.py @@ -5,6 +5,7 @@ import ldap from django_auth_ldap.config import LDAPSearch from zerver.lib.db import TimeTrackingConnection + from .config import DEPLOY_ROOT, get_from_file_if_exists from .settings import ( AUTHENTICATION_BACKENDS, diff --git a/zproject/test_settings.py b/zproject/test_settings.py index 28522199e6..0f10b85840 100644 --- a/zproject/test_settings.py +++ b/zproject/test_settings.py @@ -1,4 +1,5 @@ import os + # test_settings.py works differently from # dev_settings.py/prod_settings.py; it actually is directly referenced # by the test suite as DJANGO_SETTINGS_MODULE and imports settings.py diff --git a/zproject/urls.py b/zproject/urls.py index a3f6bc1ba7..8349b02c26 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -1,49 +1,51 @@ -from django.conf import settings -from django.conf.urls import url, include -from django.conf.urls.i18n import i18n_patterns -from django.views.generic import TemplateView, RedirectView -from django.utils.module_loading import import_string import os + +from django.conf import settings +from django.conf.urls import include, url +from django.conf.urls.i18n import i18n_patterns +from django.contrib.auth.views import ( + LoginView, + PasswordResetCompleteView, + PasswordResetConfirmView, + PasswordResetDoneView, +) +from django.utils.module_loading import import_string +from django.views.generic import RedirectView, TemplateView + import zerver.forms -from zproject import dev_urls -from zproject.legacy_urls import legacy_urls -from zerver.views.documentation import IntegrationView, MarkdownDirectoryView -from zerver.lib.integrations import WEBHOOK_INTEGRATIONS - - -from django.contrib.auth.views import (LoginView, PasswordResetDoneView, - PasswordResetConfirmView, PasswordResetCompleteView) - import zerver.tornado.views import zerver.views -import zerver.views.auth import zerver.views.archive +import zerver.views.auth import zerver.views.camo import zerver.views.compatibility -import zerver.views.home -import zerver.views.email_mirror -import zerver.views.registration -import zerver.views.portico -import zerver.views.zephyr -import zerver.views.users -import zerver.views.unsubscribe +import zerver.views.digest import zerver.views.documentation +import zerver.views.email_mirror +import zerver.views.home +import zerver.views.messages +import zerver.views.muting +import zerver.views.portico +import zerver.views.realm +import zerver.views.realm_export +import zerver.views.registration +import zerver.views.streams +import zerver.views.unsubscribe +import zerver.views.upload import zerver.views.user_groups import zerver.views.user_settings -import zerver.views.muting -import zerver.views.streams -import zerver.views.realm -import zerver.views.digest -import zerver.views.messages -import zerver.views.realm_export -import zerver.views.upload +import zerver.views.users import zerver.views.video_calls - +import zerver.views.zephyr +from zerver.lib.integrations import WEBHOOK_INTEGRATIONS from zerver.lib.rest import rest_dispatch +from zerver.views.documentation import IntegrationView, MarkdownDirectoryView +from zproject import dev_urls +from zproject.legacy_urls import legacy_urls if settings.TWO_FACTOR_AUTHENTICATION_ENABLED: - from two_factor.urls import urlpatterns as tf_urls from two_factor.gateways.twilio.urls import urlpatterns as tf_twilio_urls + from two_factor.urls import urlpatterns as tf_urls # NB: There are several other pieces of code which route requests by URL: # diff --git a/zproject/wsgi.py b/zproject/wsgi.py index 0d1253c779..75144d5732 100644 --- a/zproject/wsgi.py +++ b/zproject/wsgi.py @@ -25,6 +25,7 @@ setup_path() os.environ.setdefault("DJANGO_SETTINGS_MODULE", "zproject.settings") import django + try: django.setup() except Exception as e: @@ -45,10 +46,12 @@ except Exception as e: # need to import zerver.models first before the middleware tries to import it. import zerver.models + zerver.models # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. from django.core.wsgi import get_wsgi_application + application = get_wsgi_application() diff --git a/zthumbor/loaders/helpers.py b/zthumbor/loaders/helpers.py index 615e4d77c8..b595256dec 100644 --- a/zthumbor/loaders/helpers.py +++ b/zthumbor/loaders/helpers.py @@ -4,7 +4,7 @@ from __future__ import absolute_import import os import re import sys -from typing import Any, Text, Tuple, Optional +from typing import Any, Optional, Text, Tuple ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(ZULIP_PATH) diff --git a/zthumbor/loaders/zloader.py b/zthumbor/loaders/zloader.py index 98bd506d77..74f694273a 100644 --- a/zthumbor/loaders/zloader.py +++ b/zthumbor/loaders/zloader.py @@ -1,20 +1,24 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html from __future__ import absolute_import -from six.moves import urllib -from tornado.concurrent import return_future -from thumbor.loaders import LoaderResult, file_loader, https_loader -from tc_aws.loaders import s3_loader -from thumbor.context import Context -from .helpers import ( - separate_url_and_source_type, - THUMBOR_S3_TYPE, THUMBOR_LOCAL_FILE_TYPE, THUMBOR_EXTERNAL_TYPE, -) - -from typing import Any, Callable - import base64 import logging +from typing import Any, Callable + +from six.moves import urllib +from tornado.concurrent import return_future + +from tc_aws.loaders import s3_loader +from thumbor.context import Context +from thumbor.loaders import LoaderResult, file_loader, https_loader + +from .helpers import ( + THUMBOR_EXTERNAL_TYPE, + THUMBOR_LOCAL_FILE_TYPE, + THUMBOR_S3_TYPE, + separate_url_and_source_type, +) + def get_not_found_result(): # type: () -> LoaderResult