diff --git a/analytics/tests/test_counts.py b/analytics/tests/test_counts.py index 02765c7b00..c8ffd2b291 100644 --- a/analytics/tests/test_counts.py +++ b/analytics/tests/test_counts.py @@ -77,8 +77,8 @@ from zerver.models import ( UserActivityInterval, UserGroup, UserProfile, - get_client, ) +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.users import get_user, is_cross_realm_bot_email from zilencer.models import ( diff --git a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness index 3c3a94ecd9..3d82924b98 100755 --- a/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness +++ b/puppet/zulip_ops/files/nagios_plugins/zulip_zephyr_mirror/check_user_zephyr_mirror_liveness @@ -28,7 +28,8 @@ django.setup() from typing import Dict -from zerver.models import UserActivity, get_client +from zerver.models import UserActivity +from zerver.models.clients import get_client states: Dict[str, int] = { "OK": 0, diff --git a/zerver/actions/message_send.py b/zerver/actions/message_send.py index 7afc85d857..01c7289e0a 100644 --- a/zerver/actions/message_send.py +++ b/zerver/actions/message_send.py @@ -102,9 +102,9 @@ from zerver.models import ( UserPresence, UserProfile, UserTopic, - get_client, query_for_ids, ) +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.recipients import get_huddle_user_ids from zerver.models.streams import get_stream, get_stream_by_id_in_realm diff --git a/zerver/actions/presence.py b/zerver/actions/presence.py index 83b720e72b..00a9d8a014 100644 --- a/zerver/actions/presence.py +++ b/zerver/actions/presence.py @@ -12,7 +12,8 @@ from zerver.lib.presence import ( from zerver.lib.queue import queue_json_publish from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.users import get_user_ids_who_can_access_user -from zerver.models import Client, UserPresence, UserProfile, get_client +from zerver.models import Client, UserPresence, UserProfile +from zerver.models.clients import get_client from zerver.models.users import active_user_ids from zerver.tornado.django_api import send_event diff --git a/zerver/actions/user_settings.py b/zerver/actions/user_settings.py index a8127ec684..1c0405dff5 100644 --- a/zerver/actions/user_settings.py +++ b/zerver/actions/user_settings.py @@ -39,8 +39,8 @@ from zerver.models import ( ScheduledMessageNotificationEmail, UserPresence, UserProfile, - get_client, ) +from zerver.models.clients import get_client from zerver.models.users import bot_owner_user_ids, get_user_profile_by_id from zerver.tornado.django_api import send_event, send_event_on_commit diff --git a/zerver/decorator.py b/zerver/decorator.py index aef1b84afb..7da88ab351 100644 --- a/zerver/decorator.py +++ b/zerver/decorator.py @@ -61,7 +61,8 @@ from zerver.lib.timestamp import datetime_to_timestamp, timestamp_to_datetime from zerver.lib.users import is_2fa_verified from zerver.lib.utils import has_api_key_format from zerver.lib.webhooks.common import notify_bot_owner_about_invalid_json -from zerver.models import UserProfile, get_client +from zerver.models import UserProfile +from zerver.models.clients import get_client from zerver.models.users import get_user_profile_by_api_key if TYPE_CHECKING: diff --git a/zerver/lib/cache_helpers.py b/zerver/lib/cache_helpers.py index d83c2a40bb..7d96cb7ad1 100644 --- a/zerver/lib/cache_helpers.py +++ b/zerver/lib/cache_helpers.py @@ -24,7 +24,8 @@ from zerver.lib.cache import ( from zerver.lib.safe_session_cached_db import SessionStore from zerver.lib.sessions import session_engine from zerver.lib.users import get_all_api_keys -from zerver.models import Client, UserProfile, get_client_cache_key +from zerver.models import Client, UserProfile +from zerver.models.clients import get_client_cache_key def user_cache_items( diff --git a/zerver/lib/email_mirror.py b/zerver/lib/email_mirror.py index 13631e6bf4..b156487d35 100644 --- a/zerver/lib/email_mirror.py +++ b/zerver/lib/email_mirror.py @@ -29,15 +29,8 @@ from zerver.lib.rate_limiter import RateLimitedObject from zerver.lib.send_email import FromAddress from zerver.lib.string_validation import is_character_printable from zerver.lib.upload import upload_message_attachment -from zerver.models import ( - Message, - MissedMessageEmailAddress, - Realm, - Recipient, - Stream, - UserProfile, - get_client, -) +from zerver.models import Message, MissedMessageEmailAddress, Realm, Recipient, Stream, UserProfile +from zerver.models.clients import get_client from zerver.models.streams import get_stream_by_id_in_realm from zerver.models.users import get_system_bot, get_user_profile_by_id from zproject.backends import is_user_active diff --git a/zerver/lib/management.py b/zerver/lib/management.py index e9de2e826e..70f3e94eb2 100644 --- a/zerver/lib/management.py +++ b/zerver/lib/management.py @@ -13,7 +13,8 @@ from django.db.models import Q, QuerySet from typing_extensions import override from zerver.lib.initial_password import initial_password -from zerver.models import Client, Realm, UserProfile, get_client +from zerver.models import Client, Realm, UserProfile +from zerver.models.clients import get_client def is_integer_string(val: str) -> bool: diff --git a/zerver/lib/outgoing_webhook.py b/zerver/lib/outgoing_webhook.py index dd2a653ac8..b52c6f3629 100644 --- a/zerver/lib/outgoing_webhook.py +++ b/zerver/lib/outgoing_webhook.py @@ -19,14 +19,8 @@ from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.queue import retry_event from zerver.lib.topic import get_topic_from_message_info from zerver.lib.url_encoding import near_message_url -from zerver.models import ( - GENERIC_INTERFACE, - SLACK_INTERFACE, - Realm, - Service, - UserProfile, - get_client, -) +from zerver.models import GENERIC_INTERFACE, SLACK_INTERFACE, Realm, Service, UserProfile +from zerver.models.clients import get_client from zerver.models.users import get_user_profile_by_id diff --git a/zerver/lib/server_initialization.py b/zerver/lib/server_initialization.py index 33789bd4d0..e8a0fd40ae 100644 --- a/zerver/lib/server_initialization.py +++ b/zerver/lib/server_initialization.py @@ -11,8 +11,8 @@ from zerver.models import ( RealmAuthenticationMethod, RealmUserDefault, UserProfile, - get_client, ) +from zerver.models.clients import get_client from zerver.models.users import get_system_bot from zproject.backends import all_implemented_backend_names diff --git a/zerver/lib/test_helpers.py b/zerver/lib/test_helpers.py index 42c947b09f..ac655fa260 100644 --- a/zerver/lib/test_helpers.py +++ b/zerver/lib/test_helpers.py @@ -52,15 +52,8 @@ from zerver.lib.per_request_cache import flush_per_request_caches from zerver.lib.rate_limiter import RateLimitedIPAddr, rules from zerver.lib.request import RequestNotes from zerver.lib.upload.s3 import S3UploadBackend -from zerver.models import ( - Client, - Message, - RealmUserDefault, - Subscription, - UserMessage, - UserProfile, - get_client, -) +from zerver.models import Client, Message, RealmUserDefault, Subscription, UserMessage, UserProfile +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream from zerver.tornado.handlers import AsyncDjangoHandler, allocate_handler_id diff --git a/zerver/models/__init__.py b/zerver/models/__init__.py index 1347c9065b..ac7d841e50 100644 --- a/zerver/models/__init__.py +++ b/zerver/models/__init__.py @@ -1,7 +1,6 @@ # https://github.com/typeddjango/django-stubs/issues/1698 # mypy: disable-error-code="explicit-override" -import hashlib import time from datetime import timedelta from typing import Any, Callable, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union @@ -27,10 +26,8 @@ from django.utils.translation import gettext_lazy from django_stubs_ext import StrPromise, ValuesQuerySet from typing_extensions import override -from zerver.lib import cache from zerver.lib.cache import ( cache_delete, - cache_with_key, flush_message, flush_submessage, flush_used_upload_space_cache, @@ -59,6 +56,7 @@ from zerver.lib.validator import ( check_url, validate_select_field, ) +from zerver.models.clients import Client as Client from zerver.models.constants import MAX_TOPIC_NAME_LENGTH from zerver.models.groups import GroupGroupMembership as GroupGroupMembership from zerver.models.groups import UserGroup as UserGroup @@ -143,74 +141,6 @@ def query_for_ids( return query -class Client(models.Model): - MAX_NAME_LENGTH = 30 - name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True, unique=True) - - @override - def __str__(self) -> str: - return self.name - - def default_read_by_sender(self) -> bool: - """Used to determine whether a message was sent by a full Zulip UI - style client (and thus whether the message should be treated - as sent by a human and automatically marked as read for the - sender). The purpose of this distinction is to ensure that - message sent to the user by e.g. a Google Calendar integration - using the user's own API key don't get marked as read - automatically. - """ - sending_client = self.name.lower() - - return ( - sending_client - in ( - "zulipandroid", - "zulipios", - "zulipdesktop", - "zulipmobile", - "zulipelectron", - "zulipterminal", - "snipe", - "website", - "ios", - "android", - ) - or "desktop app" in sending_client - # Since the vast majority of messages are sent by humans - # in Zulip, treat test suite messages as such. - or (sending_client == "test suite" and settings.TEST_SUITE) - ) - - -get_client_cache: Dict[str, Client] = {} - - -def clear_client_cache() -> None: # nocoverage - global get_client_cache - get_client_cache = {} - - -def get_client(name: str) -> Client: - # Accessing KEY_PREFIX through the module is necessary - # because we need the updated value of the variable. - cache_name = cache.KEY_PREFIX + name[0 : Client.MAX_NAME_LENGTH] - if cache_name not in get_client_cache: - result = get_client_remote_cache(name) - get_client_cache[cache_name] = result - return get_client_cache[cache_name] - - -def get_client_cache_key(name: str) -> str: - return f"get_client:{hashlib.sha1(name.encode()).hexdigest()}" - - -@cache_with_key(get_client_cache_key, timeout=3600 * 24 * 7) -def get_client_remote_cache(name: str) -> Client: - (client, _) = Client.objects.get_or_create(name=name[0 : Client.MAX_NAME_LENGTH]) - return client - - class AbstractMessage(models.Model): sender = models.ForeignKey(UserProfile, on_delete=CASCADE) diff --git a/zerver/models/clients.py b/zerver/models/clients.py new file mode 100644 index 0000000000..4f788ea6d3 --- /dev/null +++ b/zerver/models/clients.py @@ -0,0 +1,77 @@ +import hashlib +from typing import Dict + +from django.conf import settings +from django.db import models +from typing_extensions import override + +from zerver.lib import cache +from zerver.lib.cache import cache_with_key + + +class Client(models.Model): + MAX_NAME_LENGTH = 30 + name = models.CharField(max_length=MAX_NAME_LENGTH, db_index=True, unique=True) + + @override + def __str__(self) -> str: + return self.name + + def default_read_by_sender(self) -> bool: + """Used to determine whether a message was sent by a full Zulip UI + style client (and thus whether the message should be treated + as sent by a human and automatically marked as read for the + sender). The purpose of this distinction is to ensure that + message sent to the user by e.g. a Google Calendar integration + using the user's own API key don't get marked as read + automatically. + """ + sending_client = self.name.lower() + + return ( + sending_client + in ( + "zulipandroid", + "zulipios", + "zulipdesktop", + "zulipmobile", + "zulipelectron", + "zulipterminal", + "snipe", + "website", + "ios", + "android", + ) + or "desktop app" in sending_client + # Since the vast majority of messages are sent by humans + # in Zulip, treat test suite messages as such. + or (sending_client == "test suite" and settings.TEST_SUITE) + ) + + +get_client_cache: Dict[str, Client] = {} + + +def clear_client_cache() -> None: # nocoverage + global get_client_cache + get_client_cache = {} + + +def get_client(name: str) -> Client: + # Accessing KEY_PREFIX through the module is necessary + # because we need the updated value of the variable. + cache_name = cache.KEY_PREFIX + name[0 : Client.MAX_NAME_LENGTH] + if cache_name not in get_client_cache: + result = get_client_remote_cache(name) + get_client_cache[cache_name] = result + return get_client_cache[cache_name] + + +def get_client_cache_key(name: str) -> str: + return f"get_client:{hashlib.sha1(name.encode()).hexdigest()}" + + +@cache_with_key(get_client_cache_key, timeout=3600 * 24 * 7) +def get_client_remote_cache(name: str) -> Client: + (client, _) = Client.objects.get_or_create(name=name[0 : Client.MAX_NAME_LENGTH]) + return client diff --git a/zerver/tests/test_decorators.py b/zerver/tests/test_decorators.py index 02dd5b62be..6cbab146e0 100644 --- a/zerver/tests/test_decorators.py +++ b/zerver/tests/test_decorators.py @@ -52,7 +52,8 @@ from zerver.lib.user_agent import parse_user_agent from zerver.lib.users import get_api_key from zerver.lib.utils import generate_api_key, has_api_key_format from zerver.middleware import LogRequests, parse_client -from zerver.models import Client, Realm, UserProfile, clear_client_cache +from zerver.models import Client, Realm, UserProfile +from zerver.models.clients import clear_client_cache from zerver.models.realms import get_realm from zerver.models.users import get_user diff --git a/zerver/tests/test_delete_unclaimed_attachments.py b/zerver/tests/test_delete_unclaimed_attachments.py index f151d2d667..807db51d36 100644 --- a/zerver/tests/test_delete_unclaimed_attachments.py +++ b/zerver/tests/test_delete_unclaimed_attachments.py @@ -13,7 +13,8 @@ from zerver.actions.scheduled_messages import check_schedule_message, delete_sch from zerver.actions.uploads import do_delete_old_unclaimed_attachments from zerver.lib.retention import clean_archived_data from zerver.lib.test_classes import UploadSerializeMixin, ZulipTestCase -from zerver.models import ArchivedAttachment, Attachment, Message, UserProfile, get_client +from zerver.models import ArchivedAttachment, Attachment, Message, UserProfile +from zerver.models.clients import get_client class UnclaimedAttachmentTest(UploadSerializeMixin, ZulipTestCase): diff --git a/zerver/tests/test_event_system.py b/zerver/tests/test_event_system.py index d1c0a3a5a8..3e478a8663 100644 --- a/zerver/tests/test_event_system.py +++ b/zerver/tests/test_event_system.py @@ -28,7 +28,8 @@ from zerver.lib.test_helpers import ( stub_event_queue_user_events, ) from zerver.lib.users import get_api_key, get_users_for_api -from zerver.models import CustomProfileField, UserMessage, UserPresence, UserProfile, get_client +from zerver.models import CustomProfileField, UserMessage, UserPresence, UserProfile +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream from zerver.models.users import get_system_bot diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index e561c6d24b..b67c921ea5 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -240,8 +240,8 @@ from zerver.models import ( UserProfile, UserStatus, UserTopic, - get_client, ) +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.streams import get_stream from zerver.models.users import get_user_by_delivery_email diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 1a6f5a7cb0..adef08fd51 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -83,8 +83,8 @@ from zerver.models import ( UserProfile, UserStatus, UserTopic, - get_client, ) +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.realms import get_realm from zerver.models.recipients import get_huddle_hash diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index 1c6e1485b7..56d9c8a9a6 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -60,15 +60,8 @@ from zerver.lib.message import render_markdown from zerver.lib.per_request_cache import flush_per_request_caches from zerver.lib.test_classes import ZulipTestCase from zerver.lib.tex import render_tex -from zerver.models import ( - Message, - RealmEmoji, - RealmFilter, - UserGroup, - UserMessage, - UserProfile, - get_client, -) +from zerver.models import Message, RealmEmoji, RealmFilter, UserGroup, UserMessage, UserProfile +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.linkifiers import linkifiers_for_realm from zerver.models.realms import get_realm diff --git a/zerver/tests/test_message_topics.py b/zerver/tests/test_message_topics.py index 3be23cb8c0..a9f0b714b2 100644 --- a/zerver/tests/test_message_topics.py +++ b/zerver/tests/test_message_topics.py @@ -6,7 +6,8 @@ from zerver.actions.streams import do_change_stream_permission from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import timeout_mock from zerver.lib.timeout import TimeoutExpiredError -from zerver.models import Message, UserMessage, get_client +from zerver.models import Message, UserMessage +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream diff --git a/zerver/tests/test_mirror_users.py b/zerver/tests/test_mirror_users.py index 133521f646..2f7e9e461a 100644 --- a/zerver/tests/test_mirror_users.py +++ b/zerver/tests/test_mirror_users.py @@ -8,7 +8,8 @@ from zerver.actions.message_send import create_mirror_user_if_needed from zerver.lib.create_user import create_user_profile from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import reset_email_visibility_to_everyone_in_zulip_realm -from zerver.models import UserProfile, get_client +from zerver.models import UserProfile +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.users import get_user from zerver.views.message_send import InvalidMirrorInputError, create_mirrored_message_users diff --git a/zerver/tests/test_push_notifications.py b/zerver/tests/test_push_notifications.py index 6a8261a575..45c901fa09 100644 --- a/zerver/tests/test_push_notifications.py +++ b/zerver/tests/test_push_notifications.py @@ -90,8 +90,8 @@ from zerver.models import ( UserMessage, UserProfile, UserTopic, - get_client, ) +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream from zilencer.models import RemoteZulipServerAuditLog diff --git a/zerver/tests/test_queue_worker.py b/zerver/tests/test_queue_worker.py index f798821558..8677f6d931 100644 --- a/zerver/tests/test_queue_worker.py +++ b/zerver/tests/test_queue_worker.py @@ -31,8 +31,8 @@ from zerver.models import ( ScheduledMessageNotificationEmail, UserActivity, UserProfile, - get_client, ) +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream from zerver.tornado.event_queue import build_offline_notification diff --git a/zerver/tests/test_retention.py b/zerver/tests/test_retention.py index 531b648c25..f4c2b5b329 100644 --- a/zerver/tests/test_retention.py +++ b/zerver/tests/test_retention.py @@ -37,8 +37,8 @@ from zerver.models import ( Stream, SubMessage, UserMessage, - get_client, ) +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.streams import get_stream from zerver.models.users import get_system_bot diff --git a/zerver/tests/test_user_status.py b/zerver/tests/test_user_status.py index 90658c7b35..492e347501 100644 --- a/zerver/tests/test_user_status.py +++ b/zerver/tests/test_user_status.py @@ -4,7 +4,8 @@ import orjson from zerver.lib.test_classes import ZulipTestCase from zerver.lib.user_status import UserInfoDict, get_user_status_dict, update_user_status -from zerver.models import UserProfile, UserStatus, get_client +from zerver.models import UserProfile, UserStatus +from zerver.models.clients import get_client def user_status_info(user: UserProfile, acting_user: Optional[UserProfile] = None) -> UserInfoDict: diff --git a/zerver/tests/test_users.py b/zerver/tests/test_users.py index d65ba87981..87036625c8 100644 --- a/zerver/tests/test_users.py +++ b/zerver/tests/test_users.py @@ -73,8 +73,8 @@ from zerver.models import ( UserProfile, UserTopic, check_valid_user_ids, - get_client, ) +from zerver.models.clients import get_client from zerver.models.groups import SystemGroups from zerver.models.prereg_users import filter_to_valid_prereg_users from zerver.models.realms import InvalidFakeEmailDomainError, get_fake_email_domain, get_realm diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index 5b34aaa7a2..2f01aae247 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -20,7 +20,8 @@ from zerver.lib.validator import ( check_string, to_non_negative_int, ) -from zerver.models import Client, UserProfile, get_client +from zerver.models import Client, UserProfile +from zerver.models.clients import get_client from zerver.models.users import get_user_profile_by_id from zerver.tornado.descriptors import is_current_port from zerver.tornado.event_queue import access_client_descriptor, fetch_events, process_notification diff --git a/zerver/views/development/cache.py b/zerver/views/development/cache.py index 94e6f663bf..5b0acc595a 100644 --- a/zerver/views/development/cache.py +++ b/zerver/views/development/cache.py @@ -7,7 +7,7 @@ from zerver.decorator import require_post from zerver.lib.cache import get_cache_backend from zerver.lib.per_request_cache import flush_per_request_caches from zerver.lib.response import json_success -from zerver.models import clear_client_cache +from zerver.models.clients import clear_client_cache ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../") diff --git a/zerver/webhooks/bitbucket2/tests.py b/zerver/webhooks/bitbucket2/tests.py index 437a511ecc..3d5c76a5ab 100644 --- a/zerver/webhooks/bitbucket2/tests.py +++ b/zerver/webhooks/bitbucket2/tests.py @@ -4,7 +4,7 @@ from zerver.lib.request import RequestNotes from zerver.lib.test_classes import WebhookTestCase from zerver.lib.test_helpers import HostRequestMock from zerver.lib.validator import wrap_wild_value -from zerver.models import get_client +from zerver.models.clients import get_client from zerver.webhooks.bitbucket2.view import get_user_info TOPIC = "Repository name" diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index 043e535535..97d59bb511 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -105,8 +105,8 @@ from zerver.models import ( UserMessage, UserProfile, get_bot_services, - get_client, ) +from zerver.models.clients import get_client from zerver.models.prereg_users import filter_to_valid_prereg_users from zerver.models.users import get_system_bot, get_user_profile_by_id diff --git a/zilencer/management/commands/populate_db.py b/zilencer/management/commands/populate_db.py index d45dcd1c4e..c6ec4b0588 100644 --- a/zilencer/management/commands/populate_db.py +++ b/zilencer/management/commands/populate_db.py @@ -65,8 +65,8 @@ from zerver.models import ( UserPresence, UserProfile, flush_alert_word, - get_client, ) +from zerver.models.clients import get_client from zerver.models.realms import get_realm from zerver.models.recipients import get_or_create_huddle from zerver.models.streams import get_stream