diff --git a/api_docs/changelog.md b/api_docs/changelog.md index ed78b5c0c1..893fd69c52 100644 --- a/api_docs/changelog.md +++ b/api_docs/changelog.md @@ -30,6 +30,8 @@ format used by the Zulip server that they are interacting with. `zulip_message_ids`. * Mobile push notification payloads for FCM to for new messages no longer contain the (unused) `content_truncated` boolean field. +- E2EE mobile push notification payloads now have a [modernized and + documented format](/api/mobile-notifications). **Feature level 412** diff --git a/api_docs/include/rest-endpoints.md b/api_docs/include/rest-endpoints.md index c696498c04..8bf227d31c 100644 --- a/api_docs/include/rest-endpoints.md +++ b/api_docs/include/rest-endpoints.md @@ -157,6 +157,7 @@ * [Fetch an API key (development only)](/api/dev-fetch-api-key) * [Send a test notification to mobile device(s)](/api/test-notify) * [Register E2EE push device](/api/register-push-device) +* [Mobile notifications](/api/mobile-notifications) * [Add an APNs device token](/api/add-apns-token) * [Remove an APNs device token](/api/remove-apns-token) * [Add an FCM registration token](/api/add-fcm-token) diff --git a/api_docs/mobile-notifications.md b/api_docs/mobile-notifications.md new file mode 100644 index 0000000000..5bb13b285d --- /dev/null +++ b/api_docs/mobile-notifications.md @@ -0,0 +1,112 @@ +# Mobile notifications + +Zulip Server 11.0+ supports end-to-end encryption (E2EE) for mobile +push notifications. Mobile push notifications sent by all Zulip +servers go through Zulip's mobile push notifications service, which +then delivers the notifications through the appropriate +platform-specific push notification service (Google's FCM or Apple's +APNs). E2EE push notifications ensure that mobile notification message +content and metadata is not visible to intermediaries. + +Mobile clients that have [registered an E2EE push +device](/api/register-push-device) will receive mobile notifications +end-to-end encrypted by their Zulip server. + +This page documents the format of the encrypted JSON-format payloads +that the client will receive through this protocol. The same encrypted +payload formats are used for both Firebase Cloud Messaging (FCM) and +Apple Push Notification service (APNs). + +## Payload examples + +### New channel message + +Sample JSON data that gets encrypted: +```json +{ + "channel_id": 10, + "channel_name": "Denmark", + "content": "@test_user_group", + "mentioned_user_group_id": 41, + "mentioned_user_group_name": "test_user_group", + "message_id": 45, + "realm_name": "Zulip Dev", + "realm_url": "http://zulip.testserver", + "recipient_type": "channel", + "sender_avatar_url": "https://secure.gravatar.com/avatar/818c212b9f8830dfef491b3f7da99a14?d=identicon&version=1", + "sender_full_name": "aaron", + "sender_id": 6, + "time": 1754385395, + "topic": "test", + "type": "message", + "user_id": 10 +} +``` + +- The `mentioned_user_group_id` and `mentioned_user_group_name` fields + are only present for messages that mention a group containing the + current user, and triggered a mobile notification because of that + group mention. For example, messages that mention both the user + directly and a group containing the user, these fields will not be + present in the payload, because the direct mention has precedence. + +**Changes**: New in Zulip 11.0 (feature level 413). + +### New direct message + +Sample JSON data that gets encrypted: +```json +{ + "content": "test content", + "message_id": 46, + "pm_users": "6,10,12,15" + "realm_name": "Zulip Dev", + "realm_url": "http://zulip.testserver", + "recipient_type": "direct", + "sender_avatar_url": "https://secure.gravatar.com/avatar/818c212b9f8830dfef491b3f7da99a14?d=identicon&version=1", + "sender_full_name": "aaron", + "sender_id": 6, + "time": 1754385290, + "type": "message", + "user_id": 10 +} +``` + +- **Group direct messages**: The `pm_users` string field is only +present for group direct messages, containing a sorted comma-separated +list of all user IDs in the group direct message conversation, +including both `user_id` and `sender_id`. + +**Changes**: New in Zulip 11.0 (feature level 413). + +### New group direct message + +### Remove notifications + +When a batch of messages that had previously been included in mobile +notifications are marked as read, are deleted, become inaccessible, or +otherwise should no longer be displayed to the user, a removal +notification is sent. + +Sample JSON data that gets encrypted: +```json +{ + "message_ids": [ + 31, + 32 + ], + "realm_name": "Zulip Dev", + "realm_url": "http://zulip.testserver", + "type": "remove", + "user_id": 10 +} +``` + +[zulip-bouncer]: https://zulip.readthedocs.io/en/latest/production/mobile-push-notifications.html#mobile-push-notification-service + +**Changes**: New in Zulip 11.0 (feature level 413). + +## Future work + +This page will eventually also document the formats of the APNs and +FCM payloads wrapping the encrypted content. diff --git a/version.py b/version.py index 833ae5417d..6f35e90a34 100644 --- a/version.py +++ b/version.py @@ -34,7 +34,7 @@ DESKTOP_WARNING_VERSION = "5.9.3" # new level means in api_docs/changelog.md, as well as "**Changes**" # entries in the endpoint's documentation in `zulip.yaml`. -API_FEATURE_LEVEL = 412 +API_FEATURE_LEVEL = 413 # Bump the minor PROVISION_VERSION to indicate that folks should provision # only when going from an old version of the code to a newer version. Bump diff --git a/zerver/lib/push_notifications.py b/zerver/lib/push_notifications.py index a9284cc893..73e632bec9 100644 --- a/zerver/lib/push_notifications.py +++ b/zerver/lib/push_notifications.py @@ -962,14 +962,15 @@ def truncate_content(content: str) -> tuple[str, bool]: return content[:200] + "…", True -def get_base_payload(user_profile: UserProfile) -> dict[str, Any]: +def get_base_payload(user_profile: UserProfile, for_legacy_clients: bool = True) -> dict[str, Any]: """Common fields for all notification payloads.""" data: dict[str, Any] = {} # These will let the app support logging into multiple realms and servers. - data["server"] = settings.EXTERNAL_HOST - data["realm_id"] = user_profile.realm.id - data["realm_uri"] = user_profile.realm.url + if for_legacy_clients: + data["server"] = settings.EXTERNAL_HOST + data["realm_id"] = user_profile.realm.id + data["realm_uri"] = user_profile.realm.url data["realm_url"] = user_profile.realm.url data["realm_name"] = user_profile.realm.name data["user_id"] = user_profile.id @@ -991,22 +992,25 @@ def get_message_payload( mentioned_user_group_id: int | None = None, mentioned_user_group_name: str | None = None, can_access_sender: bool = True, + for_legacy_clients: bool = True, ) -> dict[str, Any]: """Common fields for `message` payloads, for all platforms.""" - data = get_base_payload(user_profile) + data = get_base_payload(user_profile, for_legacy_clients) # `sender_id` is preferred, but some existing versions use `sender_email`. data["sender_id"] = message.sender.id - if not can_access_sender: - # A guest user can only receive a stream message from an - # inaccessible user as we allow unsubscribed users to send - # messages to streams. For direct messages, the guest gains - # access to the user if they where previously inaccessible. - data["sender_email"] = Address( - username=f"user{message.sender.id}", domain=get_fake_email_domain(message.realm.host) - ).addr_spec - else: - data["sender_email"] = message.sender.email + if for_legacy_clients: + if not can_access_sender: + # A guest user can only receive a stream message from an + # inaccessible user as we allow unsubscribed users to send + # messages to streams. For direct messages, the guest gains + # access to the user if they where previously inaccessible. + data["sender_email"] = Address( + username=f"user{message.sender.id}", + domain=get_fake_email_domain(message.realm.host), + ).addr_spec + else: + data["sender_email"] = message.sender.email if mentioned_user_group_id is not None: assert mentioned_user_group_name is not None @@ -1014,12 +1018,21 @@ def get_message_payload( data["mentioned_user_group_name"] = mentioned_user_group_name if message.recipient.type == Recipient.STREAM: - data["recipient_type"] = "stream" - data["stream"] = get_message_stream_name_from_database(message) - data["stream_id"] = message.recipient.type_id + channel_id = message.recipient.type_id + channel_name = get_message_stream_name_from_database(message) + + if for_legacy_clients: + data["recipient_type"] = "stream" + data["stream"] = channel_name + data["stream_id"] = channel_id + else: + data["recipient_type"] = "channel" + data["channel_name"] = channel_name + data["channel_id"] = channel_id + data["topic"] = get_topic_display_name(message.topic_name(), user_profile.default_language) elif message.recipient.type == Recipient.DIRECT_MESSAGE_GROUP: - data["recipient_type"] = "private" + data["recipient_type"] = "private" if for_legacy_clients else "direct" # For group DMs, we need to fetch the users for the pm_users field. # Note that this doesn't do a separate database query, because both # functions use the get_display_recipient_by_id cache. @@ -1027,7 +1040,7 @@ def get_message_payload( if len(recipients) > 2: data["pm_users"] = direct_message_group_users(message.recipient.id) else: # Recipient.PERSONAL - data["recipient_type"] = "private" + data["recipient_type"] = "private" if for_legacy_clients else "direct" return data @@ -1161,10 +1174,16 @@ def get_message_payload_gcm( mentioned_user_group_id: int | None = None, mentioned_user_group_name: str | None = None, can_access_sender: bool = True, + for_legacy_clients: bool = True, ) -> tuple[dict[str, Any], dict[str, Any]]: """A `message` payload + options, for Android via FCM.""" data = get_message_payload( - user_profile, message, mentioned_user_group_id, mentioned_user_group_name, can_access_sender + user_profile, + message, + mentioned_user_group_id, + mentioned_user_group_name, + can_access_sender, + for_legacy_clients, ) if not can_access_sender: @@ -1178,12 +1197,17 @@ def get_message_payload_gcm( sender_avatar_url = absolute_avatar_url(message.sender) sender_name = message.sender.full_name + if for_legacy_clients: + data["event"] = "message" + data["zulip_message_id"] = message.id # message_id is reserved for CCS + else: + data["type"] = "message" + data["message_id"] = message.id + assert message.rendered_content is not None with override_language(user_profile.default_language): content, unused = truncate_content(get_mobile_push_content(message.rendered_content)) data.update( - event="message", - zulip_message_id=message.id, # message_id is reserved for CCS time=datetime_to_timestamp(message.date_sent), content=content, sender_full_name=sender_name, @@ -1193,16 +1217,39 @@ def get_message_payload_gcm( return data, gcm_options +def get_payload_data_to_encrypt( + user_profile: UserProfile, + message: Message, + mentioned_user_group_id: int | None = None, + mentioned_user_group_name: str | None = None, + can_access_sender: bool = True, +) -> dict[str, Any]: + payload_data_to_encrypt, unused = get_message_payload_gcm( + user_profile, + message, + mentioned_user_group_id, + mentioned_user_group_name, + can_access_sender, + for_legacy_clients=False, + ) + return payload_data_to_encrypt + + def get_remove_payload_gcm( user_profile: UserProfile, message_ids: list[int], + for_legacy_clients: bool = True, ) -> tuple[dict[str, Any], dict[str, Any]]: """A `remove` payload + options, for Android via FCM.""" - gcm_payload = get_base_payload(user_profile) - gcm_payload.update( - event="remove", - zulip_message_ids=",".join(str(id) for id in message_ids), - ) + gcm_payload = get_base_payload(user_profile, for_legacy_clients) + + if for_legacy_clients: + gcm_payload["event"] = "remove" + gcm_payload["zulip_message_ids"] = ",".join(str(id) for id in message_ids) + else: + gcm_payload["type"] = "remove" + gcm_payload["message_ids"] = message_ids + gcm_options = {"priority": "normal"} return gcm_payload, gcm_options @@ -1221,6 +1268,16 @@ def get_remove_payload_apns(user_profile: UserProfile, message_ids: list[int]) - return apns_data +def get_remove_payload_data_to_encrypt( + user_profile: UserProfile, + message_ids: list[int], +) -> dict[str, Any]: + payload_data_to_encrypt, unused = get_remove_payload_gcm( + user_profile, message_ids, for_legacy_clients=False + ) + return payload_data_to_encrypt + + def handle_remove_push_notification(user_profile_id: int, message_ids: list[int]) -> None: """This should be called when a message that previously had a mobile push notification executed is read. This triggers a push to the @@ -1263,12 +1320,15 @@ def handle_remove_push_notification(user_profile_id: int, message_ids: list[int] truncated_message_ids = sorted(message_ids)[-MAX_APNS_MESSAGE_IDS:] gcm_payload, gcm_options = get_remove_payload_gcm(user_profile, truncated_message_ids) apns_payload = get_remove_payload_apns(user_profile, truncated_message_ids) + payload_data_to_encrypt = get_remove_payload_data_to_encrypt( + user_profile, truncated_message_ids + ) # We need to call both the legacy/non-E2EE and E2EE functions # for sending mobile notifications, since we don't at this time # know which mobile app version the user may be using. send_push_notifications_legacy(user_profile, apns_payload, gcm_payload, gcm_options) - send_push_notifications(user_profile, apns_payload, gcm_payload, is_removal=True) + send_push_notifications(user_profile, payload_data_to_encrypt, is_removal=True) # We intentionally use the non-truncated message_ids here. We are # assuming in this very rare case that the user has manually @@ -1402,8 +1462,7 @@ def get_encrypted_data(payload_data_to_encrypt: dict[str, Any], public_key_str: def send_push_notifications( user_profile: UserProfile, - apns_payload_data_to_encrypt: dict[str, Any], - fcm_payload_data_to_encrypt: dict[str, Any], + payload_data_to_encrypt: dict[str, Any], is_removal: bool = False, ) -> None: # Uses 'zerver_pushdevice_user_bouncer_device_id_idx' index. @@ -1426,15 +1485,12 @@ def send_push_notifications( push_requests: list[FCMPushRequest | APNsPushRequest] = [] for push_device in push_devices: assert push_device.bouncer_device_id is not None # for mypy + encrypted_data = get_encrypted_data(payload_data_to_encrypt, push_device.push_public_key) if push_device.token_kind == PushDevice.TokenKind.APNS: apns_http_headers = APNsHTTPHeaders( apns_priority=apns_priority, apns_push_type=apns_push_type, ) - encrypted_data = get_encrypted_data( - apns_payload_data_to_encrypt, - push_device.push_public_key, - ) apns_payload = APNsPayload( push_account_id=push_device.push_account_id, encrypted_data=encrypted_data, @@ -1446,10 +1502,6 @@ def send_push_notifications( ) push_requests.append(apns_push_request) else: - encrypted_data = get_encrypted_data( - fcm_payload_data_to_encrypt, - push_device.push_public_key, - ) fcm_payload = PushRequestBasePayload( push_account_id=push_device.push_account_id, encrypted_data=encrypted_data, @@ -1667,13 +1719,16 @@ def handle_push_notification(user_profile_id: int, missed_message: dict[str, Any gcm_payload, gcm_options = get_message_payload_gcm( user_profile, message, mentioned_user_group_id, mentioned_user_group_name, can_access_sender ) + payload_data_to_encrypt = get_payload_data_to_encrypt( + user_profile, message, mentioned_user_group_id, mentioned_user_group_name, can_access_sender + ) logger.info("Sending push notifications to mobile clients for user %s", user_profile_id) # We need to call both the legacy/non-E2EE and E2EE functions # for sending mobile notifications, since we don't at this time # know which mobile app version the user may be using. send_push_notifications_legacy(user_profile, apns_payload, gcm_payload, gcm_options) - send_push_notifications(user_profile, apns_payload, gcm_payload) + send_push_notifications(user_profile, payload_data_to_encrypt) def send_test_push_notification_directly_to_devices( diff --git a/zerver/tests/test_e2ee_push_notifications.py b/zerver/tests/test_e2ee_push_notifications.py index ca8a40a0fe..de7cacd9a6 100644 --- a/zerver/tests/test_e2ee_push_notifications.py +++ b/zerver/tests/test_e2ee_push_notifications.py @@ -2,15 +2,21 @@ from datetime import datetime, timezone from unittest import mock import responses +import time_machine from django.test import override_settings +from django.utils.timezone import now from firebase_admin.exceptions import InternalError from firebase_admin.messaging import UnregisteredError from analytics.models import RealmCount +from zerver.actions.user_groups import check_add_user_group +from zerver.lib.avatar import absolute_avatar_url from zerver.lib.push_notifications import handle_push_notification, handle_remove_push_notification from zerver.lib.test_classes import E2EEPushNotificationTestCase from zerver.lib.test_helpers import activate_push_notification_service +from zerver.lib.timestamp import datetime_to_timestamp from zerver.models import PushDevice, UserMessage +from zerver.models.realms import get_realm from zerver.models.scheduled_jobs import NotificationTriggers from zilencer.models import RemoteRealm, RemoteRealmCount @@ -598,6 +604,83 @@ class SendPushNotificationTest(E2EEPushNotificationTestCase): ) self.assertEqual(realm_count_dict, dict(subgroup=None, value=4)) + def test_payload_data_to_encrypt_channel_message(self) -> None: + hamlet = self.example_user("hamlet") + aaron = self.example_user("aaron") + realm = get_realm("zulip") + user_group = check_add_user_group(realm, "test_user_group", [hamlet], acting_user=hamlet) + + time_now = now() + self.subscribe(aaron, "Denmark") + with time_machine.travel(time_now, tick=False): + message_id = self.send_stream_message( + sender=aaron, + stream_name="Denmark", + content=f"@*{user_group.name}*", + skip_capture_on_commit_callbacks=True, + ) + missed_message = { + "message_id": message_id, + "trigger": NotificationTriggers.MENTION, + "mentioned_user_group_id": user_group.id, + } + + expected_payload_data_to_encrypt = { + "realm_url": realm.url, + "realm_name": realm.name, + "user_id": hamlet.id, + "sender_id": aaron.id, + "mentioned_user_group_id": user_group.id, + "mentioned_user_group_name": user_group.name, + "recipient_type": "channel", + "channel_name": "Denmark", + "channel_id": self.get_stream_id("Denmark"), + "topic": "test", + "type": "message", + "message_id": message_id, + "time": datetime_to_timestamp(time_now), + "content": f"@{user_group.name}", + "sender_full_name": aaron.full_name, + "sender_avatar_url": absolute_avatar_url(aaron), + } + with mock.patch("zerver.lib.push_notifications.send_push_notifications") as m: + handle_push_notification(hamlet.id, missed_message) + + self.assertEqual(m.call_args.args[1], expected_payload_data_to_encrypt) + + def test_payload_data_to_encrypt_direct_message(self) -> None: + hamlet = self.example_user("hamlet") + aaron = self.example_user("aaron") + realm = get_realm("zulip") + + time_now = now() + with time_machine.travel(time_now, tick=False): + message_id = self.send_personal_message( + from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True + ) + missed_message = { + "message_id": message_id, + "trigger": NotificationTriggers.DIRECT_MESSAGE, + } + + expected_payload_data_to_encrypt = { + "realm_url": realm.url, + "realm_name": realm.name, + "user_id": hamlet.id, + "sender_id": aaron.id, + "recipient_type": "direct", + "type": "message", + "message_id": message_id, + "time": datetime_to_timestamp(time_now), + "content": "test content", + "sender_full_name": aaron.full_name, + "sender_avatar_url": absolute_avatar_url(aaron), + } + with mock.patch("zerver.lib.push_notifications.send_push_notifications") as m: + handle_push_notification(hamlet.id, missed_message) + + self.assertEqual(m.call_args.args[1], expected_payload_data_to_encrypt) + @activate_push_notification_service() class RemovePushNotificationTest(E2EEPushNotificationTestCase): @@ -679,6 +762,30 @@ class RemovePushNotificationTest(E2EEPushNotificationTestCase): zerver_logger.output[2], ) + def test_remove_payload_data_to_encrypt(self) -> None: + hamlet = self.example_user("hamlet") + aaron = self.example_user("aaron") + realm = get_realm("zulip") + + message_id_one = self.send_personal_message( + from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True + ) + message_id_two = self.send_personal_message( + from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True + ) + + expected_payload_data_to_encrypt = { + "realm_url": realm.url, + "realm_name": realm.name, + "user_id": hamlet.id, + "type": "remove", + "message_ids": [message_id_one, message_id_two], + } + with mock.patch("zerver.lib.push_notifications.send_push_notifications") as m: + handle_remove_push_notification(hamlet.id, [message_id_one, message_id_two]) + + self.assertEqual(m.call_args.args[1], expected_payload_data_to_encrypt) + class RequireE2EEPushNotificationsSettingTest(E2EEPushNotificationTestCase): def test_content_redacted(self) -> None: @@ -722,8 +829,7 @@ class RequireE2EEPushNotificationsSettingTest(E2EEPushNotificationTestCase): self.assertEqual(mock_legacy.call_args.args[2]["content"], "New message") mock_e2ee.assert_called_once() - self.assertEqual(mock_e2ee.call_args.args[1]["alert"]["body"], "not-redacted") - self.assertEqual(mock_e2ee.call_args.args[2]["content"], "not-redacted") + self.assertEqual(mock_e2ee.call_args.args[1]["content"], "not-redacted") message_id = self.send_personal_message( from_user=aaron, to_user=hamlet, skip_capture_on_commit_callbacks=True