diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 75fba6d7a7..fa3fca9de6 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -3,11 +3,12 @@ import copy import logging import time -from collections.abc import Callable, Collection, Iterable, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Sequence from typing import Any from django.conf import settings from django.utils.translation import gettext as _ +from typing_extensions import NotRequired, TypedDict from version import API_FEATURE_LEVEL, ZULIP_MERGE_BASE, ZULIP_VERSION from zerver.actions.default_streams import default_stream_groups_to_dicts_sorted @@ -1639,6 +1640,20 @@ def apply_event( raise AssertionError("Unexpected event type {}".format(event["type"])) +class ClientCapabilities(TypedDict): + # This field was accidentally made required when it was added in v2.0.0-781; + # this was not realized until after the release of Zulip 2.1.2. (It remains + # required to help ensure backwards compatibility of client code.) + notification_settings_null: bool + # Any new fields of `client_capabilities` should be optional. Add them here. + bulk_message_deletion: NotRequired[bool] + user_avatar_url_field_optional: NotRequired[bool] + stream_typing_notifications: NotRequired[bool] + user_settings_object: NotRequired[bool] + linkifier_url_template: NotRequired[bool] + user_list_incomplete: NotRequired[bool] + + def do_events_register( user_profile: UserProfile | None, realm: Realm, @@ -1652,7 +1667,7 @@ def do_events_register( all_public_streams: bool = False, include_subscribers: bool = True, include_streams: bool = True, - client_capabilities: Mapping[str, bool] = {}, + client_capabilities: ClientCapabilities = ClientCapabilities(notification_settings_null=False), narrow: Collection[NarrowTerm] = [], fetch_event_types: Collection[str] | None = None, spectator_requested_language: str | None = None, diff --git a/zerver/lib/home.py b/zerver/lib/home.py index 8d29e75fd3..47dcd5bc37 100644 --- a/zerver/lib/home.py +++ b/zerver/lib/home.py @@ -8,7 +8,7 @@ from django.utils import translation from two_factor.utils import default_device from zerver.context_processors import get_apps_page_url -from zerver.lib.events import do_events_register +from zerver.lib.events import ClientCapabilities, do_events_register from zerver.lib.i18n import ( get_and_set_request_language, get_language_list, @@ -147,15 +147,16 @@ def build_page_params_for_home_page_load( The page_params data structure gets sent to the client. """ - client_capabilities = { - "notification_settings_null": True, - "bulk_message_deletion": True, - "user_avatar_url_field_optional": True, - "stream_typing_notifications": True, - "user_settings_object": True, - "linkifier_url_template": True, - "user_list_incomplete": True, - } + + client_capabilities = ClientCapabilities( + notification_settings_null=True, + bulk_message_deletion=True, + user_avatar_url_field_optional=True, + stream_typing_notifications=True, + user_settings_object=True, + linkifier_url_template=True, + user_list_incomplete=True, + ) if user_profile is not None: client = RequestNotes.get_notes(request).client diff --git a/zerver/tests/test_event_queue.py b/zerver/tests/test_event_queue.py index 8ca7ed2f19..dbde312b03 100644 --- a/zerver/tests/test_event_queue.py +++ b/zerver/tests/test_event_queue.py @@ -155,7 +155,9 @@ class MissedMessageHookTest(ZulipTestCase): return access_client_descriptor(user.id, queue_id) def destroy_event_queue(self, user: UserProfile, queue_id: str) -> None: - result = self.tornado_call(cleanup_event_queue, user, {"queue_id": queue_id}) + # Ignore this when running mypy, since mypy expects queue_id to be specified when calling the function + # but it is actually extracted from the request body using the typed_endpoint decorator. + result = self.tornado_call(cleanup_event_queue, user, {"queue_id": queue_id}) # type: ignore[arg-type] self.assert_json_success(result) def assert_maybe_enqueue_notifications_call_args( diff --git a/zerver/tests/test_openapi.py b/zerver/tests/test_openapi.py index fd51f89c3d..8cc2e23379 100644 --- a/zerver/tests/test_openapi.py +++ b/zerver/tests/test_openapi.py @@ -318,7 +318,8 @@ so maybe we shouldn't mark it as intentionally undocumented in the URLs. val = 6 for t in types: if isinstance(t, tuple): - return t # e.g. (list, dict) or (list, str) + # e.g. (list, dict) or (list, str) + return t # nocoverage v = priority.get(t, 6) if v < val: val = v @@ -343,8 +344,6 @@ so maybe we shouldn't mark it as intentionally undocumented in the URLs. elif origin in [list, abc.Sequence]: [st] = get_args(t) return (list, self.get_standardized_argument_type(st)) - elif origin in [dict, abc.Mapping]: - return dict raise AssertionError(f"Unknown origin {origin}") def render_openapi_type_exception( diff --git a/zerver/tornado/views.py b/zerver/tornado/views.py index 1543767b34..94e4710ce3 100644 --- a/zerver/tornado/views.py +++ b/zerver/tornado/views.py @@ -1,29 +1,21 @@ import time -from collections.abc import Callable, Mapping, Sequence -from typing import Any, TypeVar +from collections.abc import Callable +from typing import Annotated, Any, TypeVar from asgiref.sync import async_to_sync from django.conf import settings from django.http import HttpRequest, HttpResponse from django.utils.translation import gettext as _ -from pydantic import Json +from pydantic import BaseModel, Json, NonNegativeInt, StringConstraints, model_validator from typing_extensions import ParamSpec from zerver.decorator import internal_api_view, process_client from zerver.lib.exceptions import JsonableError from zerver.lib.queue import get_queue_client -from zerver.lib.request import REQ, RequestNotes, has_request_variables +from zerver.lib.request import RequestNotes from zerver.lib.response import AsynchronousResponse, json_success -from zerver.lib.typed_endpoint import typed_endpoint -from zerver.lib.validator import ( - check_bool, - check_dict, - check_int, - check_list, - check_string, - to_non_negative_int, -) -from zerver.models import Client, UserProfile +from zerver.lib.typed_endpoint import ApiParamConfig, DocumentationStatus, typed_endpoint +from zerver.models import UserProfile from zerver.models.clients import get_client from zerver.models.users import get_user_profile_by_id from zerver.tornado.descriptors import is_current_port @@ -47,10 +39,8 @@ def in_tornado_thread(f: Callable[P, T]) -> Callable[P, T]: @internal_api_view(True) -@has_request_variables -def notify( - request: HttpRequest, data: Mapping[str, Any] = REQ(json_validator=check_dict([])) -) -> HttpResponse: +@typed_endpoint +def notify(request: HttpRequest, *, data: Json[dict[str, Any]]) -> HttpResponse: # Only the puppeteer full-stack tests use this endpoint; it # injects an event, as if read from RabbitMQ. in_tornado_thread(process_notification)(data) @@ -77,9 +67,9 @@ def web_reload_clients( ) -@has_request_variables +@typed_endpoint def cleanup_event_queue( - request: HttpRequest, user_profile: UserProfile, queue_id: str = REQ() + request: HttpRequest, user_profile: UserProfile, *, queue_id: str ) -> HttpResponse: log_data = RequestNotes.get_notes(request).log_data assert log_data is not None @@ -108,10 +98,8 @@ def cleanup_event_queue( @internal_api_view(True) -@has_request_variables -def get_events_internal( - request: HttpRequest, user_profile_id: int = REQ(json_validator=check_int) -) -> HttpResponse: +@typed_endpoint +def get_events_internal(request: HttpRequest, *, user_profile_id: Json[int]) -> HttpResponse: user_profile = get_user_profile_by_id(user_profile_id) RequestNotes.get_notes(request).requester_for_logs = user_profile.format_requester_for_logs() assert is_current_port(get_user_tornado_port(user_profile)) @@ -137,64 +125,90 @@ def get_events(request: HttpRequest, user_profile: UserProfile) -> HttpResponse: return get_events_backend(request, user_profile) -@has_request_variables +class UserClient(BaseModel): + id: int + name: Annotated[str, StringConstraints(max_length=30)] + + @model_validator(mode="before") + @classmethod + def convert_term(cls, elem: str) -> dict[str, Any]: + client = get_client(elem) + return {"id": client.id, "name": client.name} + + +@typed_endpoint def get_events_backend( request: HttpRequest, user_profile: UserProfile, + *, # user_client is intended only for internal Django=>Tornado requests # and thus shouldn't be documented for external use. - user_client: Client | None = REQ( - converter=lambda var_name, s: get_client(s), default=None, intentionally_undocumented=True - ), - last_event_id: int | None = REQ(json_validator=check_int, default=None), - queue_id: str | None = REQ(default=None), + user_client: Annotated[ + UserClient | None, + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = None, + last_event_id: Json[int] | None = None, + queue_id: str | None = None, # apply_markdown, client_gravatar, all_public_streams, and various # other parameters are only used when registering a new queue via this # endpoint. This is a feature used primarily by get_events_internal # and not expected to be used by third-party clients. - apply_markdown: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - client_gravatar: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - slim_presence: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - all_public_streams: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - event_types: Sequence[str] | None = REQ( - default=None, json_validator=check_list(check_string), intentionally_undocumented=True - ), - dont_block: bool = REQ(default=False, json_validator=check_bool), - narrow: Sequence[Sequence[str]] = REQ( - default=[], - json_validator=check_list(check_list(check_string)), - intentionally_undocumented=True, - ), - lifespan_secs: int = REQ( - default=0, converter=to_non_negative_int, intentionally_undocumented=True - ), - bulk_message_deletion: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - stream_typing_notifications: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - user_settings_object: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - pronouns_field_type_supported: bool = REQ( - default=True, json_validator=check_bool, intentionally_undocumented=True - ), - linkifier_url_template: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), - user_list_incomplete: bool = REQ( - default=False, json_validator=check_bool, intentionally_undocumented=True - ), + apply_markdown: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + client_gravatar: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + slim_presence: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + all_public_streams: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + event_types: Annotated[ + Json[list[str]] | None, + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = None, + dont_block: Json[bool] = False, + narrow: Annotated[ + Json[list[list[str]]] | None, + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = None, + lifespan_secs: Annotated[ + Json[NonNegativeInt], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = 0, + bulk_message_deletion: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + stream_typing_notifications: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + user_settings_object: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + pronouns_field_type_supported: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = True, + linkifier_url_template: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, + user_list_incomplete: Annotated[ + Json[bool], + ApiParamConfig(documentation_status=DocumentationStatus.INTENTIONALLY_UNDOCUMENTED), + ] = False, ) -> HttpResponse: + if narrow is None: + narrow = [] if all_public_streams and not user_profile.can_access_public_streams(): raise JsonableError(_("User not authorized for this query")) @@ -205,8 +219,9 @@ def get_events_backend( if user_client is None: valid_user_client = RequestNotes.get_notes(request).client assert valid_user_client is not None + valid_user_client_name = valid_user_client.name else: - valid_user_client = user_client + valid_user_client_name = user_client.name new_queue_data = None if queue_id is None: @@ -214,7 +229,7 @@ def get_events_backend( user_profile_id=user_profile.id, realm_id=user_profile.realm_id, event_types=event_types, - client_type_name=valid_user_client.name, + client_type_name=valid_user_client_name, apply_markdown=apply_markdown, client_gravatar=client_gravatar, slim_presence=slim_presence, @@ -234,7 +249,7 @@ def get_events_backend( user_profile_id=user_profile.id, queue_id=queue_id, last_event_id=last_event_id, - client_type_name=valid_user_client.name, + client_type_name=valid_user_client_name, dont_block=dont_block, handler_id=handler_id, new_queue_data=new_queue_data, diff --git a/zerver/views/events_register.py b/zerver/views/events_register.py index 1a438efb05..bdc7d20795 100644 --- a/zerver/views/events_register.py +++ b/zerver/views/events_register.py @@ -1,19 +1,20 @@ -from collections.abc import Sequence -from typing import TypeAlias +from typing import Annotated, TypeAlias +from annotated_types import Len from django.conf import settings from django.contrib.auth.models import AnonymousUser from django.http import HttpRequest, HttpResponse from django.utils.translation import gettext as _ +from pydantic import Json from zerver.context_processors import get_valid_realm_from_request from zerver.lib.compatibility import is_pronouns_field_type_supported -from zerver.lib.events import do_events_register +from zerver.lib.events import ClientCapabilities, do_events_register from zerver.lib.exceptions import JsonableError, MissingAuthenticationError from zerver.lib.narrow_helpers import narrow_dataclasses_from_tuples -from zerver.lib.request import REQ, RequestNotes, has_request_variables +from zerver.lib.request import RequestNotes from zerver.lib.response import json_success -from zerver.lib.validator import check_bool, check_dict, check_int, check_list, check_string +from zerver.lib.typed_endpoint import ApiParamConfig, DocumentationStatus, typed_endpoint from zerver.models import Stream, UserProfile @@ -24,59 +25,36 @@ def _default_all_public_streams(user_profile: UserProfile, all_public_streams: b return user_profile.default_all_public_streams -def _default_narrow( - user_profile: UserProfile, narrow: Sequence[Sequence[str]] -) -> Sequence[Sequence[str]]: +def _default_narrow(user_profile: UserProfile, narrow: list[list[str]]) -> list[list[str]]: default_stream: Stream | None = user_profile.default_events_register_stream if not narrow and default_stream is not None: narrow = [["stream", default_stream.name]] return narrow -NarrowT: TypeAlias = Sequence[Sequence[str]] +NarrowT: TypeAlias = list[Annotated[list[str], Len(min_length=2, max_length=2)]] -@has_request_variables +@typed_endpoint def events_register_backend( request: HttpRequest, maybe_user_profile: UserProfile | AnonymousUser, - apply_markdown: bool = REQ(default=False, json_validator=check_bool), - client_gravatar_raw: bool | None = REQ( - "client_gravatar", default=None, json_validator=check_bool - ), - slim_presence: bool = REQ(default=False, json_validator=check_bool), - all_public_streams: bool | None = REQ(default=None, json_validator=check_bool), - include_subscribers: bool = REQ(default=False, json_validator=check_bool), - client_capabilities: dict[str, bool] | None = REQ( - json_validator=check_dict( - [ - # This field was accidentally made required when it was added in v2.0.0-781; - # this was not realized until after the release of Zulip 2.1.2. (It remains - # required to help ensure backwards compatibility of client code.) - ("notification_settings_null", check_bool), - ], - [ - # Any new fields of `client_capabilities` should be optional. Add them here. - ("bulk_message_deletion", check_bool), - ("user_avatar_url_field_optional", check_bool), - ("stream_typing_notifications", check_bool), - ("user_settings_object", check_bool), - ("linkifier_url_template", check_bool), - ("user_list_incomplete", check_bool), - ], - value_validator=check_bool, - ), - default=None, - ), - event_types: Sequence[str] | None = REQ(json_validator=check_list(check_string), default=None), - fetch_event_types: Sequence[str] | None = REQ( - json_validator=check_list(check_string), default=None - ), - narrow: NarrowT = REQ( - json_validator=check_list(check_list(check_string, length=2)), default=[] - ), - queue_lifespan_secs: int = REQ(json_validator=check_int, default=0, documentation_pending=True), + *, + apply_markdown: Json[bool] = False, + client_gravatar_raw: Annotated[Json[bool | None], ApiParamConfig("client_gravatar")] = None, + slim_presence: Json[bool] = False, + all_public_streams: Json[bool] | None = None, + include_subscribers: Json[bool] = False, + client_capabilities: Json[ClientCapabilities] | None = None, + event_types: Json[list[str]] | None = None, + fetch_event_types: Json[list[str]] | None = None, + narrow: Json[NarrowT] | None = None, + queue_lifespan_secs: Annotated[ + Json[int], ApiParamConfig(documentation_status=DocumentationStatus.DOCUMENTATION_PENDING) + ] = 0, ) -> HttpResponse: + if narrow is None: + narrow = [] if client_gravatar_raw is None: client_gravatar = maybe_user_profile.is_authenticated else: @@ -121,7 +99,7 @@ def events_register_backend( include_streams = False if client_capabilities is None: - client_capabilities = {} + client_capabilities = ClientCapabilities(notification_settings_null=False) client = RequestNotes.get_notes(request).client assert client is not None