From 5f8edf669d95d9a11d36a8fe4cf12dd9e4b1f2b7 Mon Sep 17 00:00:00 2001 From: Prakhar Pratyush Date: Mon, 2 Jun 2025 22:03:06 +0530 Subject: [PATCH] zerver: Add endpoint to register a push device to server. This commit adds an endpoint to register a push device to receive E2EE push notifications. --- api_docs/changelog.md | 10 + api_docs/include/rest-endpoints.md | 1 + api_docs/unmerged.d/ZF-1653f2.md | 2 - version.py | 2 +- zerver/lib/event_schema.py | 2 + zerver/lib/event_types.py | 7 + zerver/lib/events.py | 3 + zerver/lib/push_registration.py | 138 +++++++++ zerver/lib/remote_server.py | 8 + zerver/models/push_notifications.py | 3 + zerver/openapi/python_examples.py | 18 ++ zerver/openapi/zulip.yaml | 152 +++++++++- zerver/tests/test_events.py | 36 +++ zerver/tests/test_push_registration.py | 270 ++++++++++++++++++ zerver/views/push_notifications.py | 62 +++- .../missedmessage_mobile_notifications.py | 6 +- zproject/urls.py | 2 + 17 files changed, 711 insertions(+), 11 deletions(-) delete mode 100644 api_docs/unmerged.d/ZF-1653f2.md create mode 100644 zerver/lib/push_registration.py diff --git a/api_docs/changelog.md b/api_docs/changelog.md index e4d241bbc0..82630c815f 100644 --- a/api_docs/changelog.md +++ b/api_docs/changelog.md @@ -20,6 +20,16 @@ format used by the Zulip server that they are interacting with. ## Changes in Zulip 11.0 +**Feature level 406** + +* [`POST /register`](/api/register-queue): Added `push_devices` + field to response. +* [`GET /events`](/api/get-events): A `push_device` event is sent + to clients when registration to bouncer either succeeds or fails. +* [`POST /mobile_push/register`](/api/register-push-device): Added + an endpoint to register a device to receive end-to-end encrypted + mobile push notifications. + **Feature level 405** * [Message formatting](/api/message-formatting): Added new HTML diff --git a/api_docs/include/rest-endpoints.md b/api_docs/include/rest-endpoints.md index ce1fcd11a5..014b446102 100644 --- a/api_docs/include/rest-endpoints.md +++ b/api_docs/include/rest-endpoints.md @@ -155,6 +155,7 @@ * [Fetch an API key (production)](/api/fetch-api-key) * [Fetch an API key (development only)](/api/dev-fetch-api-key) * [Send a test notification to mobile device(s)](/api/test-notify) +* [Register E2EE push device](/api/register-push-device) * [Add an APNs device token](/api/add-apns-token) * [Remove an APNs device token](/api/remove-apns-token) * [Add an FCM registration token](/api/add-fcm-token) diff --git a/api_docs/unmerged.d/ZF-1653f2.md b/api_docs/unmerged.d/ZF-1653f2.md deleted file mode 100644 index 6d7d7f7af8..0000000000 --- a/api_docs/unmerged.d/ZF-1653f2.md +++ /dev/null @@ -1,2 +0,0 @@ -* [`POST /register`](/api/register-queue): Added `push_devices` - field to response. diff --git a/version.py b/version.py index 8fb5574176..a73c6bdc08 100644 --- a/version.py +++ b/version.py @@ -34,7 +34,7 @@ DESKTOP_WARNING_VERSION = "5.9.3" # new level means in api_docs/changelog.md, as well as "**Changes**" # entries in the endpoint's documentation in `zulip.yaml`. -API_FEATURE_LEVEL = 405 +API_FEATURE_LEVEL = 406 # Bump the minor PROVISION_VERSION to indicate that folks should provision # only when going from an old version of the code to a newer version. Bump diff --git a/zerver/lib/event_schema.py b/zerver/lib/event_schema.py index 99d301e77b..d30b5fb235 100644 --- a/zerver/lib/event_schema.py +++ b/zerver/lib/event_schema.py @@ -44,6 +44,7 @@ from zerver.lib.event_types import ( EventNavigationViewUpdate, EventOnboardingSteps, EventPresence, + EventPushDevice, EventReactionAdd, EventReactionRemove, EventRealmBilling, @@ -186,6 +187,7 @@ check_navigation_view_add = make_checker(EventNavigationViewAdd) check_navigation_view_remove = make_checker(EventNavigationViewRemove) check_navigation_view_update = make_checker(EventNavigationViewUpdate) check_onboarding_steps = make_checker(EventOnboardingSteps) +check_push_device = make_checker(EventPushDevice) check_reaction_add = make_checker(EventReactionAdd) check_reaction_remove = make_checker(EventReactionRemove) check_realm_billing = make_checker(EventRealmBilling) diff --git a/zerver/lib/event_types.py b/zerver/lib/event_types.py index 78d7457302..dfb0d40849 100644 --- a/zerver/lib/event_types.py +++ b/zerver/lib/event_types.py @@ -283,6 +283,13 @@ class EventOnboardingSteps(BaseEvent): onboarding_steps: list[OnboardingSteps] +class EventPushDevice(BaseEvent): + type: Literal["push_device"] + push_account_id: str + status: Literal["active", "failed", "pending"] + error_code: str | None + + class NavigationViewFields(BaseModel): fragment: str is_pinned: bool diff --git a/zerver/lib/events.py b/zerver/lib/events.py index 5475b548c5..971770013d 100644 --- a/zerver/lib/events.py +++ b/zerver/lib/events.py @@ -1957,6 +1957,9 @@ def apply_event( elif event["type"] == "restart": # The Tornado process restarted. This has no effect; we ignore it. pass + elif event["type"] == "push_device": + state["push_devices"][event["push_account_id"]]["status"] = event["status"] + state["push_devices"][event["push_account_id"]]["error_code"] = event.get("error_code") else: raise AssertionError("Unexpected event type {}".format(event["type"])) diff --git a/zerver/lib/push_registration.py b/zerver/lib/push_registration.py new file mode 100644 index 0000000000..47b0c8eaa5 --- /dev/null +++ b/zerver/lib/push_registration.py @@ -0,0 +1,138 @@ +import logging +from typing import TypedDict + +from django.conf import settings + +from zerver.lib.exceptions import ( + InvalidBouncerPublicKeyError, + InvalidEncryptedPushRegistrationError, + JsonableError, + MissingRemoteRealmError, + RequestExpiredError, +) +from zerver.lib.push_notifications import PushNotificationsDisallowedByBouncerError +from zerver.lib.remote_server import ( + PushNotificationBouncerError, + PushNotificationBouncerRetryLaterError, + PushNotificationBouncerServerError, + send_to_push_bouncer, +) +from zerver.models import PushDevice +from zerver.models.users import UserProfile, get_user_profile_by_id +from zerver.tornado.django_api import send_event_on_commit + +if settings.ZILENCER_ENABLED: + from zilencer.views import do_register_remote_push_device + +logger = logging.getLogger(__name__) + + +class RegisterPushDeviceToBouncerQueueItem(TypedDict): + user_profile_id: int + bouncer_public_key: str + encrypted_push_registration: str + push_account_id: int + + +def handle_registration_to_bouncer_failure( + user_profile: UserProfile, push_account_id: int, error_code: str +) -> None: + """Handles a failed registration request to the bouncer by + notifying or preparing to notify clients. + + * Sends a `push_device` event to notify online clients immediately. + + * Stores the `error_code` in the `PushDevice` table. This is later + used, along with other metadata, to notify offline clients the + next time they call `/register`. See the `push_devices` field in + the `/register` response. + """ + PushDevice.objects.filter(user=user_profile, push_account_id=push_account_id).update( + error_code=error_code + ) + event = dict( + type="push_device", + push_account_id=str(push_account_id), + status="failed", + error_code=error_code, + ) + send_event_on_commit(user_profile.realm, event, [user_profile.id]) + + # Report the `REQUEST_EXPIRED_ERROR` to the server admins as it indicates + # a long-lasting outage somewhere between the server and the bouncer, + # most likely in either the server or its local network configuration. + if error_code == PushDevice.ErrorCode.REQUEST_EXPIRED: + logging.error( + "Push device registration request for user_id=%s, push_account_id=%s expired.", + user_profile.id, + push_account_id, + ) + + +def handle_register_push_device_to_bouncer(queue_item: RegisterPushDeviceToBouncerQueueItem) -> None: + user_profile_id = queue_item["user_profile_id"] + user_profile = get_user_profile_by_id(user_profile_id) + bouncer_public_key = queue_item["bouncer_public_key"] + encrypted_push_registration = queue_item["encrypted_push_registration"] + push_account_id = queue_item["push_account_id"] + + try: + if settings.ZILENCER_ENABLED: + device_id = do_register_remote_push_device( + bouncer_public_key, + encrypted_push_registration, + push_account_id, + realm=user_profile.realm, + ) + else: + post_data: dict[str, str | int] = { + "realm_uuid": str(user_profile.realm.uuid), + "push_account_id": push_account_id, + "encrypted_push_registration": encrypted_push_registration, + "bouncer_public_key": bouncer_public_key, + } + result = send_to_push_bouncer("POST", "push/e2ee/register", post_data) + assert isinstance(result["device_id"], int) # for mypy + device_id = result["device_id"] + except ( + PushNotificationBouncerRetryLaterError, + PushNotificationBouncerServerError, + ) as e: # nocoverage + # Network error or 5xx error response from bouncer server. + # Keep retrying to register until `RequestExpiredError` is raised. + raise PushNotificationBouncerRetryLaterError(e.msg) + except ( + # Need to resubmit realm info - `manage.py register_server` + MissingRemoteRealmError, + # Invalid credentials or unexpected status code + PushNotificationBouncerError, + # Plan doesn't allow sending push notifications + PushNotificationsDisallowedByBouncerError, + ): + # Server admins need to fix these set of errors, report them. + # Server should keep retrying to register until `RequestExpiredError` is raised. + error_msg = f"Push device registration request for user_id={user_profile.id}, push_account_id={push_account_id} failed." + logging.error(error_msg) + raise PushNotificationBouncerRetryLaterError(error_msg) + except ( + InvalidBouncerPublicKeyError, + InvalidEncryptedPushRegistrationError, + RequestExpiredError, + # Any future or unexpected exceptions that we add. + JsonableError, + ) as e: + handle_registration_to_bouncer_failure( + user_profile, push_account_id, error_code=e.__class__.code.name + ) + return + + # Registration successful. + PushDevice.objects.filter(user=user_profile, push_account_id=push_account_id).update( + bouncer_device_id=device_id + ) + event = dict( + type="push_device", + push_account_id=str(push_account_id), + status="active", + ) + send_event_on_commit(user_profile.realm, event, [user_profile.id]) diff --git a/zerver/lib/remote_server.py b/zerver/lib/remote_server.py index 329da72b52..dcffbcd3b1 100644 --- a/zerver/lib/remote_server.py +++ b/zerver/lib/remote_server.py @@ -21,9 +21,11 @@ from zerver.actions.realm_settings import ( ) from zerver.lib import redis_utils from zerver.lib.exceptions import ( + InvalidBouncerPublicKeyError, JsonableError, MissingRemoteRealmError, RemoteRealmServerMismatchError, + RequestExpiredError, ) from zerver.lib.outgoing_http import OutgoingSession from zerver.lib.queue import queue_event_on_commit @@ -217,6 +219,12 @@ def send_to_push_bouncer( # The callers requesting this endpoint want the exception to propagate # so they can catch it. raise RemoteRealmServerMismatchError + elif endpoint == "push/e2ee/register" and code == "INVALID_BOUNCER_PUBLIC_KEY": + raise InvalidBouncerPublicKeyError + elif endpoint == "push/e2ee/register" and code == "REQUEST_EXPIRED": + raise RequestExpiredError + elif endpoint == "push/e2ee/register" and code == "MISSING_REMOTE_REALM": + raise MissingRemoteRealmError else: # But most other errors coming from the push bouncer # server are client errors (e.g. never-registered token) diff --git a/zerver/models/push_notifications.py b/zerver/models/push_notifications.py index ef3cdefe55..eaf950d29f 100644 --- a/zerver/models/push_notifications.py +++ b/zerver/models/push_notifications.py @@ -101,6 +101,9 @@ class PushDevice(AbstractPushDevice): # generated by the client to register. But the API treats # that as idempotent request. # We treat (user, push_account_id) as a unique registration. + # + # Also, the unique index created is used by queries in `get_push_accounts`, + # `register_push_device`, and `handle_register_push_device_to_bouncer`. fields=["user", "push_account_id"], name="unique_push_device_user_push_account_id", ), diff --git a/zerver/openapi/python_examples.py b/zerver/openapi/python_examples.py index be78f3c838..1026abb6b3 100644 --- a/zerver/openapi/python_examples.py +++ b/zerver/openapi/python_examples.py @@ -1679,6 +1679,23 @@ def remove_fcm_token(client: Client) -> None: validate_against_openapi_schema(result, "/users/me/android_gcm_reg_id", "delete", "200") +@openapi_test_function("/mobile_push/register:post") +def register_push_device(client: Client) -> None: + # {code_example|start} + # Register a push device. + request = { + "token_kind": "fcm", + "push_account_id": 2408, + "push_public_key": "push-public-key", + "bouncer_public_key": "bouncer-public-key", + "encrypted_push_registration": "encrypted-push-registration-data", + } + result = client.call_endpoint(url="/mobile_push/register", method="POST", request=request) + # {code_example|end} + assert_success_response(result) + validate_against_openapi_schema(result, "/mobile_push/register", "post", "200") + + @openapi_test_function("/typing:post") def set_typing_status(client: Client) -> None: ensure_users([10, 11], ["hamlet", "iago"]) @@ -1986,6 +2003,7 @@ def test_users(client: Client, owner_client: Client) -> None: remove_apns_token(client) add_fcm_token(client) remove_fcm_token(client) + register_push_device(client) def test_streams(client: Client, nonadmin_client: Client) -> None: diff --git a/zerver/openapi/zulip.yaml b/zerver/openapi/zulip.yaml index 5c49334571..66806eec35 100644 --- a/zerver/openapi/zulip.yaml +++ b/zerver/openapi/zulip.yaml @@ -1798,6 +1798,60 @@ paths: "upload_space_used": 0, "id": 0, } + - type: object + additionalProperties: false + description: | + Event sent to a user's clients when the metadata in the + `push_devices` dictionary for the user changes. + + Helps clients to live-update the `push_devices` dictionary + returned in [`POST /register`](/api/register-queue) response. + + **Changes**: New in Zulip 11.0 (feature level 406). + properties: + id: + $ref: "#/components/schemas/EventIdSchema" + type: + allOf: + - $ref: "#/components/schemas/EventTypeSchema" + - enum: + - push_device + push_account_id: + type: string + description: | + The push account ID for this client registration. + + See [`POST /mobile_push/register`)(/api/register-push-device) + for details on push account IDs. + status: + type: string + enum: + - active + - failed + - pending + description: | + The updated registration status. Will be "active", "failed", or + "pending". + error_code: + type: string + nullable: true + description: | + If the status is `"failed"`, a [Zulip API error + code](/api/rest-error-handling) indicating the type of failure that + occurred. + + The following error codes have recommended client behavior: + + - `"INVALID_BOUNCER_PUBLIC_KEY"` - Inform the user to update app. + - `"REQUEST_EXPIRED` - Retry with a fresh payload. + If the status is "failed", an error code explaining the failure. + example: + { + "id": 1, + "type": "push_device", + "push_account_id": "1234", + "status": "active", + } - type: object additionalProperties: false description: | @@ -12293,6 +12347,93 @@ paths: oneOf: - $ref: "#/components/schemas/InvalidPushDeviceTokenError" - $ref: "#/components/schemas/InvalidRemotePushDeviceTokenError" + /mobile_push/register: + post: + operationId: register-push-device + summary: Register E2EE push device + tags: ["mobile"] + description: | + Register a device to receive end-to-end encrypted mobile push notifications. + + **Changes**: New in Zulip 11.0 (feature level 406). + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + type: object + properties: + token_kind: + description: | + Whether the token was generated by FCM or APNs. + type: string + enum: + - fcm + - apns + example: "fcm" + push_account_id: + description: | + A random integer generated by the client that will be included + in mobile push notifications along with encrypted payloads to + identify pushes as being related to this registration. + Must be unique in the client's table of accounts. + type: integer + example: 2408 + push_public_key: + description: | + Public part of the asymmetric key pair generated by the client + using NaCl (the Networking and Cryptography Library), encoded + using a RFC 4648 standard base64 encoder. + + It is used to encrypt notifications for delivery to the client. + type: string + example: "push-public-key" + bouncer_public_key: + description: | + Which of the bouncer's public keys the client used to encrypt the + `PushRegistration` dictionary. + + When the bouncer rotates the key, a new asymmetric key pair is created, + and the new public key is baked into a new client release. Because + the bouncer routinely rotates key, this field clarifies which + public key the client is using. + type: string + example: "bouncer-public-key" + encrypted_push_registration: + description: | + Ciphertext generated by encrypting a `PushRegistration` dictionary + using the `bouncer_public_key`, encoded using a RFC 4648 standard + base64 encoder. + + The `PushRegistration` dictionary contains the fields `token`, + `token_kind`, `timestamp`, and (for iOS devices) `ios_app_id`. + The dictionary is JSON-encoded before encryption. + type: string + example: "encrypted-push-registration-data" + required: + - token_kind + - push_account_id + - push_public_key + - bouncer_public_key + - encrypted_push_registration + responses: + "200": + $ref: "#/components/responses/SimpleSuccess" + "400": + description: Bad request. + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/CodedError" + - example: + { + "result": "error", + "msg": "Server is not configured to use push notification service.", + "code": "BAD_REQUEST", + } + description: | + Error when the server is not configured to use push notification service: /user_topics: post: operationId: update-user-topic @@ -17347,7 +17488,7 @@ paths: Dictionary where each entry describes the user's push device's registration status and error code (if registration failed). - **Changes**: New in Zulip 11.0 (feature level ZF-1653f2). + **Changes**: New in Zulip 11.0 (feature level 406). additionalProperties: description: | `{push_account_id}`: Dictionary containing the details of @@ -17364,12 +17505,11 @@ paths: type: string nullable: true description: | - The error code returned when registration to bouncer fails, `null` otherwise. + If the status is `"failed"`, a [Zulip API error + code](/api/rest-error-handling) indicating the type of failure that + occurred. - Either `"INVALID_BOUNCER_PUBLIC_KEY"`, `"BAD_REQUEST"`, or `"REQUEST_EXPIRED"`. - - The client should use these error codes to fix the issue which led to registration - failure and retry. + The following error codes have recommended client behavior: - `"INVALID_BOUNCER_PUBLIC_KEY"` - Inform the user to update app. - `"REQUEST_EXPIRED` - Retry with a fresh payload. diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 7c8fb1007e..faf664a647 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -189,6 +189,7 @@ from zerver.lib.event_schema import ( check_navigation_view_update, check_onboarding_steps, check_presence, + check_push_device, check_reaction_add, check_reaction_remove, check_realm_bot_add, @@ -247,9 +248,14 @@ from zerver.lib.event_schema import ( check_user_topic, ) from zerver.lib.events import apply_events, fetch_initial_state_data, post_process_state +from zerver.lib.exceptions import InvalidBouncerPublicKeyError from zerver.lib.markdown import render_message_markdown from zerver.lib.mention import MentionBackend, MentionData from zerver.lib.muted_users import get_mute_object +from zerver.lib.push_registration import ( + RegisterPushDeviceToBouncerQueueItem, + handle_register_push_device_to_bouncer, +) from zerver.lib.streams import check_update_all_streams_active_status, user_has_metadata_access from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( @@ -281,6 +287,7 @@ from zerver.models import ( MultiuseInvite, NamedUserGroup, PreregistrationUser, + PushDevice, Realm, RealmAuditLog, RealmDomain, @@ -4212,6 +4219,35 @@ class NormalActionsTest(BaseAction): self.assertEqual(audit_log.acting_user, self.user_profile) self.assertEqual(audit_log.extra_data["realm_export_id"], export_row_id) + def test_push_device_registration_failure(self) -> None: + hamlet = self.example_user("hamlet") + self.login_user(hamlet) + + push_device = PushDevice.objects.create( + user=hamlet, + token_kind=PushDevice.TokenKind.FCM, + push_account_id=2408, + push_public_key="dummy-push-public-key", + ) + + queue_item: RegisterPushDeviceToBouncerQueueItem = { + "user_profile_id": push_device.user.id, + "push_account_id": push_device.push_account_id, + "bouncer_public_key": "bouncer-public-key", + "encrypted_push_registration": "encrypted-push-registration", + } + with ( + mock.patch( + "zerver.lib.push_registration.do_register_remote_push_device", + side_effect=InvalidBouncerPublicKeyError, + ), + self.verify_action(state_change_expected=True, num_events=1) as events, + ): + handle_register_push_device_to_bouncer(queue_item) + check_push_device("events[0]", events[0]) + self.assertEqual(events[0]["status"], "failed") + self.assertEqual(events[0]["error_code"], "INVALID_BOUNCER_PUBLIC_KEY") + def test_notify_realm_export_on_failure(self) -> None: do_change_user_role( self.user_profile, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None diff --git a/zerver/tests/test_push_registration.py b/zerver/tests/test_push_registration.py index 37575da60e..8a6e9506b2 100644 --- a/zerver/tests/test_push_registration.py +++ b/zerver/tests/test_push_registration.py @@ -2,13 +2,23 @@ import uuid from datetime import timedelta import orjson +import responses from django.conf import settings +from django.test import override_settings from django.utils.timezone import now from nacl.encoding import Base64Encoder from nacl.public import PublicKey, SealedBox +from zerver.lib.exceptions import ( + InvalidBouncerPublicKeyError, + InvalidEncryptedPushRegistrationError, + RequestExpiredError, +) +from zerver.lib.queue import queue_event_on_commit from zerver.lib.test_classes import BouncerTestCase +from zerver.lib.test_helpers import activate_push_notification_service, mock_queue_publish from zerver.lib.timestamp import datetime_to_timestamp +from zerver.models import PushDevice from zilencer.models import RemotePushDevice, RemoteRealm @@ -190,3 +200,263 @@ class RegisterPushDeviceToBouncer(BouncerTestCase): invalid_token_payload, ) self.assert_json_error(result, "Invalid encrypted_push_registration") + + +class RegisterPushDeviceToServer(BouncerTestCase): + def get_register_push_device_payload( + self, + token: str = "apple-tokenaz", + token_kind: str = RemotePushDevice.TokenKind.APNS, + ios_app_id: str | None = "example.app", + timestamp: int | None = None, + ) -> dict[str, str | int]: + if timestamp is None: + timestamp = datetime_to_timestamp(now()) + + push_registration = { + "token": token, + "token_kind": token_kind, + "ios_app_id": ios_app_id, + "timestamp": timestamp, + } + + assert settings.PUSH_REGISTRATION_ENCRYPTION_KEYS + public_key_str: str = next(iter(settings.PUSH_REGISTRATION_ENCRYPTION_KEYS.keys())) + public_key = PublicKey(public_key_str.encode("utf-8"), Base64Encoder) + sealed_box = SealedBox(public_key) + encrypted_push_registration_bytes = sealed_box.encrypt( + orjson.dumps(push_registration), Base64Encoder + ) + encrypted_push_registration = encrypted_push_registration_bytes.decode("utf-8") + + payload: dict[str, str | int] = { + "token_kind": token_kind, + "push_account_id": 2408, + "push_public_key": "push-public-key", + "bouncer_public_key": public_key_str, + "encrypted_push_registration": encrypted_push_registration, + } + return payload + + @activate_push_notification_service() + @responses.activate + def test_register_push_device_success(self) -> None: + self.add_mock_response() + self.login("hamlet") + + push_devices_count = PushDevice.objects.count() + self.assertEqual(push_devices_count, 0) + + payload = self.get_register_push_device_payload() + + # Verify the created `PushDevice` row while the + # `register_push_device_to_bouncer` event is still not + # consumed by the `PushNotificationsWorker` worker. + with mock_queue_publish("zerver.views.push_notifications.queue_event_on_commit") as m: + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_success(result) + m.assert_called_once() + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertIsNone(push_devices[0].bouncer_device_id) + self.assertEqual(push_devices[0].status, "pending") + + queue_name = m.call_args[0][0] + queue_message = m.call_args[0][1] + + # Now, the `PushNotificationsWorker` worker consumes. + with self.capture_send_event_calls(expected_num_events=1) as events: + queue_event_on_commit(queue_name, queue_message) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertIsNotNone(push_devices[0].bouncer_device_id) + self.assertEqual(push_devices[0].status, "active") + self.assertEqual( + events[0]["event"], + dict(type="push_device", push_account_id="2408", status="active"), + ) + + # Idempotent + with self.capture_send_event_calls(expected_num_events=0): + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + + # For self-hosted servers. They make a network call to bouncer + # instead of a `do_register_remote_push_device` function call. + with ( + self.settings(ZILENCER_ENABLED=False), + self.capture_send_event_calls(expected_num_events=1) as events, + ): + payload["push_account_id"] = 5555 + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.order_by("pk") + self.assert_length(push_devices, 2) + self.assertIsNotNone(push_devices[1].bouncer_device_id) + self.assertEqual(push_devices[1].status, "active") + self.assertEqual( + events[0]["event"], + dict(type="push_device", push_account_id="5555", status="active"), + ) + + @override_settings(ZILENCER_ENABLED=False) + def test_server_not_configured_for_push_notification_error(self) -> None: + self.login("hamlet") + payload = self.get_register_push_device_payload() + + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_error(result, "Server is not configured to use push notification service.") + + @activate_push_notification_service() + @override_settings(ZILENCER_ENABLED=False) + @responses.activate + def test_invalid_bouncer_public_key_error(self) -> None: + self.add_mock_response() + hamlet = self.example_user("hamlet") + self.login_user(hamlet) + + push_devices_count = PushDevice.objects.count() + self.assertEqual(push_devices_count, 0) + + payload = self.get_register_push_device_payload() + + # Verify InvalidBouncerPublicKeyError + invalid_bouncer_public_key_payload = {**payload, "bouncer_public_key": "invalid public key"} + with self.capture_send_event_calls(expected_num_events=1) as events: + result = self.client_post( + "/json/mobile_push/register", invalid_bouncer_public_key_payload + ) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertEqual(push_devices[0].error_code, InvalidBouncerPublicKeyError.code.name) + self.assertEqual(push_devices[0].status, "failed") + self.assertEqual( + events[0]["event"], + dict( + type="push_device", + push_account_id="2408", + status="failed", + error_code="INVALID_BOUNCER_PUBLIC_KEY", + ), + ) + + # Retrying with correct payload results in success. + # `error_code` of the same PushDevice row updated to None. + with self.capture_send_event_calls(expected_num_events=1): + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertIsNone(push_devices[0].error_code) + self.assertEqual(push_devices[0].status, "active") + + @activate_push_notification_service() + @override_settings(ZILENCER_ENABLED=False) + @responses.activate + def test_invalid_encrypted_push_registration_error(self) -> None: + self.add_mock_response() + hamlet = self.example_user("hamlet") + self.login_user(hamlet) + + push_devices_count = PushDevice.objects.count() + self.assertEqual(push_devices_count, 0) + + # Verify InvalidEncryptedPushRegistrationError + invalid_token_payload = self.get_register_push_device_payload(token="") + with self.capture_send_event_calls(expected_num_events=1) as events: + result = self.client_post("/json/mobile_push/register", invalid_token_payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertEqual( + push_devices[0].error_code, InvalidEncryptedPushRegistrationError.code.name + ) + self.assertEqual( + events[0]["event"], + dict( + type="push_device", + push_account_id="2408", + status="failed", + error_code="BAD_REQUEST", + ), + ) + + @activate_push_notification_service() + @override_settings(ZILENCER_ENABLED=False) + @responses.activate + def test_request_expired_error(self) -> None: + self.add_mock_response() + hamlet = self.example_user("hamlet") + self.login_user(hamlet) + + push_devices_count = PushDevice.objects.count() + self.assertEqual(push_devices_count, 0) + + # Verify RequestExpiredError + liveness_timed_out_payload = self.get_register_push_device_payload( + timestamp=datetime_to_timestamp(now() - timedelta(days=2)) + ) + with ( + self.assertLogs(level="ERROR") as m, + self.capture_send_event_calls(expected_num_events=1) as events, + ): + result = self.client_post("/json/mobile_push/register", liveness_timed_out_payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + self.assertEqual(push_devices[0].error_code, RequestExpiredError.code.name) + self.assertEqual( + events[0]["event"], + dict( + type="push_device", + push_account_id="2408", + status="failed", + error_code="REQUEST_EXPIRED", + ), + ) + self.assertEqual( + m.output, + [ + f"ERROR:root:Push device registration request for user_id={hamlet.id}, push_account_id=2408 expired." + ], + ) + + @activate_push_notification_service() + @override_settings(ZILENCER_ENABLED=False) + @responses.activate + def test_missing_remote_realm_error(self) -> None: + self.add_mock_response() + hamlet = self.example_user("hamlet") + self.login_user(hamlet) + + push_devices_count = PushDevice.objects.count() + self.assertEqual(push_devices_count, 0) + + payload = self.get_register_push_device_payload() + + # Verify MissingRemoteRealm + # Update realm's UUID to a random UUID. + hamlet.realm.uuid = uuid.uuid4() + hamlet.realm.save() + + with ( + self.assertLogs(level="ERROR") as m, + self.capture_send_event_calls(expected_num_events=0), + ): + result = self.client_post("/json/mobile_push/register", payload) + self.assert_json_success(result) + push_devices = PushDevice.objects.all() + self.assert_length(push_devices, 1) + # We keep retrying until `RequestExpiredError` is raised. + self.assertEqual(push_devices[0].status, "pending") + self.assertEqual( + m.output[0], + f"ERROR:root:Push device registration request for user_id={hamlet.id}, push_account_id=2408 failed.", + ) + + # TODO: Verify that we retry for a day, then raise `RequestExpiredError`. + # This implementation would be a follow-up. Currently `retry_event` + # leads to 3 retries at max. diff --git a/zerver/views/push_notifications.py b/zerver/views/push_notifications.py index e34503dbe3..a26c3a589e 100644 --- a/zerver/views/push_notifications.py +++ b/zerver/views/push_notifications.py @@ -1,9 +1,12 @@ +from typing import Annotated + import orjson from django.conf import settings from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.urls import reverse from django.utils.translation import gettext as _ +from pydantic import Json from zerver.decorator import human_users_only, zulip_login_required from zerver.lib import redis_utils @@ -23,6 +26,8 @@ from zerver.lib.push_notifications import ( uses_notification_bouncer, validate_token, ) +from zerver.lib.push_registration import RegisterPushDeviceToBouncerQueueItem +from zerver.lib.queue import queue_event_on_commit from zerver.lib.remote_server import ( SELF_HOSTING_REGISTRATION_TAKEOVER_CHALLENGE_TOKEN_REDIS_KEY, UserDataForRemoteBilling, @@ -32,7 +37,8 @@ from zerver.lib.remote_server import ( ) from zerver.lib.response import json_success from zerver.lib.typed_endpoint import ApnsAppId, typed_endpoint, typed_endpoint_without_parameters -from zerver.models import PushDeviceToken, UserProfile +from zerver.lib.typed_endpoint_validators import check_string_in_validator +from zerver.models import PushDevice, PushDeviceToken, UserProfile from zerver.views.errors import config_error redis_client = redis_utils.get_redis_client() @@ -254,3 +260,57 @@ def self_hosting_registration_transfer_challenge_verify( verification_secret = data["verification_secret"] return json_success(request, data={"verification_secret": verification_secret}) + + +@human_users_only +@typed_endpoint +def register_push_device( + request: HttpRequest, + user_profile: UserProfile, + *, + token_kind: Annotated[str, check_string_in_validator(PushDevice.TokenKind.values)], + push_account_id: Json[int], + # Key that the client is requesting be used for + # encrypting push notifications for delivery to it. + push_public_key: str, + # Key that the client claims was used to encrypt + # `encrypted_push_registration`. + bouncer_public_key: str, + # Registration data encrypted by mobile client for bouncer. + encrypted_push_registration: str, +) -> HttpResponse: + if not (settings.ZILENCER_ENABLED or uses_notification_bouncer()): + raise JsonableError(_("Server is not configured to use push notification service.")) + + # Idempotency + already_registered = PushDevice.objects.filter( + user=user_profile, push_account_id=push_account_id, error_code__isnull=True + ).exists() + if already_registered: + return json_success(request) + + PushDevice.objects.update_or_create( + user=user_profile, + push_account_id=push_account_id, + defaults={"token_kind": token_kind, "push_public_key": push_public_key, "error_code": None}, + ) + + # We use a queue worker to make the request to the bouncer + # to complete the registration, so that any transient failures + # can be managed between the two servers, without the mobile + # device and its often-irregular network access in the picture. + queue_item: RegisterPushDeviceToBouncerQueueItem = { + "user_profile_id": user_profile.id, + "bouncer_public_key": bouncer_public_key, + "encrypted_push_registration": encrypted_push_registration, + "push_account_id": push_account_id, + } + queue_event_on_commit( + "missedmessage_mobile_notifications", + { + "type": "register_push_device_to_bouncer", + "payload": queue_item, + }, + ) + + return json_success(request) diff --git a/zerver/worker/missedmessage_mobile_notifications.py b/zerver/worker/missedmessage_mobile_notifications.py index b27444dd2a..5792e700d6 100644 --- a/zerver/worker/missedmessage_mobile_notifications.py +++ b/zerver/worker/missedmessage_mobile_notifications.py @@ -10,6 +10,7 @@ from zerver.lib.push_notifications import ( handle_remove_push_notification, initialize_push_notifications, ) +from zerver.lib.push_registration import handle_register_push_device_to_bouncer from zerver.lib.queue import retry_event from zerver.lib.remote_server import PushNotificationBouncerRetryLaterError from zerver.worker.base import QueueProcessingWorker, assign_queue @@ -46,7 +47,10 @@ class PushNotificationsWorker(QueueProcessingWorker): @override def consume(self, event: dict[str, Any]) -> None: try: - if event.get("type", "add") == "remove": + event_type = event.get("type") + if event_type == "register_push_device_to_bouncer": + handle_register_push_device_to_bouncer(event["payload"]) + elif event_type == "remove": message_ids = event["message_ids"] handle_remove_push_notification(event["user_profile_id"], message_ids) else: diff --git a/zproject/urls.py b/zproject/urls.py index ab0071a79f..5b2afd22d9 100644 --- a/zproject/urls.py +++ b/zproject/urls.py @@ -109,6 +109,7 @@ from zerver.views.presence import ( from zerver.views.push_notifications import ( add_android_reg_id, add_apns_device_token, + register_push_device, remove_android_reg_id, remove_apns_device_token, self_hosting_auth_json_endpoint, @@ -454,6 +455,7 @@ v1_api_and_json_patterns = [ ), rest_path("users/me/android_gcm_reg_id", POST=add_android_reg_id, DELETE=remove_android_reg_id), rest_path("mobile_push/test_notification", POST=send_test_push_notification_api), + rest_path("mobile_push/register", POST=register_push_device), # users/*/presence => zerver.views.presence. rest_path( "users/me/presence", POST=(update_active_status_backend, {"narrow_user_session_cache"})