register: Handle "Pronouns" type fields for older mobile clients.

Mobile clients older than v27.192 do not support PRONOUNS type
custom profile fields, so we instead change the type of it to
SHORT_TEXT in the data sent with register response and also in
the events sent to those clients.

zerver/lib/compatibility.py

@@ -138,3 +138,24 @@ def is_unsupported_browser(user_agent: str) -> Tuple[bool, Optional[str]]:
     if browser_name == "Internet Explorer":
         return (True, browser_name)
     return (False, browser_name)
+
+
+def is_pronouns_field_type_supported(user_agent_str: str) -> bool:
+    # In order to avoid users having a bad experience with these
+    # custom profile fields disappearing after applying migration
+    # 0421_migrate_pronouns_custom_profile_fields, we provide this
+    # compatibility shim to show such custom profile fields as
+    # SHORT_TEXT to older mobile app clients.
+    #
+    # TODO/compatibility(7.0): Because this is a relatively minor
+    # detail, we can remove this compatibility hack once most users
+    # have upgraded to a sufficiently new mobile client.
+    user_agent = parse_user_agent(user_agent_str)
+    if user_agent["name"] != "ZulipMobile":
+        return True
+
+    FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD = "27.192"
+    if version_lt(user_agent["version"], FIRST_VERSION_TO_SUPPORT_PRONOUNS_FIELD):
+        return False
+
+    return True

zerver/lib/events.py

@@ -114,6 +114,7 @@ def fetch_initial_state_data(
     include_subscribers: bool = True,
     include_streams: bool = True,
     spectator_requested_language: Optional[str] = None,
+    pronouns_field_type_supported: bool = True,
 ) -> Dict[str, Any]:
     """When `event_types` is None, fetches the core data powering the
     web app's `page_params` and `/api/v1/register` (for mobile/terminal
@@ -159,6 +160,13 @@ def fetch_initial_state_data(
             for item in CustomProfileField.ALL_FIELD_TYPES
         }
 
+        if not pronouns_field_type_supported:
+            for field in state["custom_profile_fields"]:
+                if field["type"] == CustomProfileField.PRONOUNS:
+                    field["type"] = CustomProfileField.SHORT_TEXT
+
+            del state["custom_profile_field_types"]["PRONOUNS"]
+
     if want("hotspots"):
         # Even if we offered special hotspots for guests without an
         # account, we'd maybe need to store their state using cookies
@@ -1372,6 +1380,7 @@ def do_events_register(
     narrow: Collection[Sequence[str]] = [],
     fetch_event_types: Optional[Collection[str]] = None,
     spectator_requested_language: Optional[str] = None,
+    pronouns_field_type_supported: bool = True,
 ) -> Dict[str, Any]:
     # Technically we don't need to check this here because
     # build_narrow_filter will check it, but it's nicer from an error
@@ -1443,6 +1452,7 @@ def do_events_register(
             bulk_message_deletion=bulk_message_deletion,
             stream_typing_notifications=stream_typing_notifications,
             user_settings_object=user_settings_object,
+            pronouns_field_type_supported=pronouns_field_type_supported,
         )
 
         if queue_id is None:
@@ -1458,6 +1468,7 @@ def do_events_register(
             slim_presence=slim_presence,
             include_subscribers=include_subscribers,
             include_streams=include_streams,
+            pronouns_field_type_supported=pronouns_field_type_supported,
         )
 
         # Apply events that came in while we were fetching initial data

zerver/tests/test_compatibility.py

@@ -1,6 +1,11 @@
 from unittest import mock
 
-from zerver.lib.compatibility import find_mobile_os, is_outdated_desktop_app, version_lt
+from zerver.lib.compatibility import (
+    find_mobile_os,
+    is_outdated_desktop_app,
+    is_pronouns_field_type_supported,
+    version_lt,
+)
 from zerver.lib.test_classes import ZulipTestCase
 
 
@@ -160,3 +165,26 @@ class CompatibilityTest(ZulipTestCase):
         )
 
         self.assertEqual(is_outdated_desktop_app(""), (False, False, False))
+
+    def test_is_pronouns_field_type_supported(self) -> None:
+        self.assertEqual(
+            is_pronouns_field_type_supported("ZulipMobile/20.0.103 (Android 6.0.1)"), False
+        )
+        self.assertEqual(is_pronouns_field_type_supported("ZulipMobile/20.0.103 (iOS 12.0)"), False)
+
+        self.assertEqual(
+            is_pronouns_field_type_supported("ZulipMobile/27.191 (Android 6.0.1)"), False
+        )
+        self.assertEqual(is_pronouns_field_type_supported("ZulipMobile/27.191 (iOS 12.0)"), False)
+
+        self.assertEqual(
+            is_pronouns_field_type_supported("ZulipMobile/27.192 (Android 6.0.1)"), True
+        )
+        self.assertEqual(is_pronouns_field_type_supported("ZulipMobile/27.192 (iOS 12.0)"), True)
+
+        self.assertEqual(
+            is_pronouns_field_type_supported(
+                "ZulipElectron/5.2.0 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Zulip/5.2.0 Chrome/80.0.3987.165 Electron/8.2.5 Safari/537.36"
+            ),
+            True,
+        )

zerver/tests/test_event_system.py

@@ -9,6 +9,7 @@ from django.test import override_settings
 from django.utils.timezone import now as timezone_now
 
 from version import API_FEATURE_LEVEL, ZULIP_MERGE_BASE, ZULIP_VERSION
+from zerver.actions.custom_profile_fields import try_update_realm_custom_profile_field
 from zerver.actions.message_send import check_send_message
 from zerver.actions.presence import do_update_user_presence
 from zerver.actions.realm_settings import do_set_realm_property
@@ -21,6 +22,7 @@ from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import HostRequestMock, dummy_handler, stub_event_queue_user_events
 from zerver.lib.users import get_api_key, get_raw_user_data
 from zerver.models import (
+    CustomProfileField,
     Realm,
     UserMessage,
     UserPresence,
@@ -515,6 +517,59 @@ class GetEventsTest(ZulipTestCase):
             )
         self.assertIn("not authorized for queue", cm.output[0])
 
+    def test_get_events_custom_profile_fields(self) -> None:
+        user_profile = self.example_user("iago")
+        self.login_user(user_profile)
+        profile_field = CustomProfileField.objects.get(realm=user_profile.realm, name="Pronouns")
+
+        def check_pronouns_type_field_supported(
+            pronouns_field_type_supported: bool, new_name: str
+        ) -> None:
+            clear_client_event_queues_for_testing()
+
+            queue_data = dict(
+                apply_markdown=True,
+                all_public_streams=True,
+                client_type_name="ZulipMobile",
+                event_types=["custom_profile_fields"],
+                last_connection_time=time.time(),
+                queue_timeout=0,
+                realm_id=user_profile.realm.id,
+                user_profile_id=user_profile.id,
+                pronouns_field_type_supported=pronouns_field_type_supported,
+            )
+
+            client = allocate_client_descriptor(queue_data)
+
+            try_update_realm_custom_profile_field(
+                realm=user_profile.realm, field=profile_field, name=new_name
+            )
+            result = self.tornado_call(
+                get_events,
+                user_profile,
+                {
+                    "queue_id": client.event_queue.id,
+                    "user_client": "ZulipAndroid",
+                    "last_event_id": -1,
+                    "dont_block": orjson.dumps(True).decode(),
+                },
+            )
+            events = orjson.loads(result.content)["events"]
+            self.assert_json_success(result)
+            self.assert_length(events, 1)
+
+            pronouns_field = [
+                field for field in events[0]["fields"] if field["id"] == profile_field.id
+            ][0]
+            if pronouns_field_type_supported:
+                expected_type = CustomProfileField.PRONOUNS
+            else:
+                expected_type = CustomProfileField.SHORT_TEXT
+            self.assertEqual(pronouns_field["type"], expected_type)
+
+        check_pronouns_type_field_supported(False, "Pronouns field")
+        check_pronouns_type_field_supported(True, "Pronouns")
+
 
 class FetchInitialStateDataTest(ZulipTestCase):
     # Non-admin users don't have access to all bots
@@ -677,6 +732,30 @@ class FetchInitialStateDataTest(ZulipTestCase):
                 self.assertIn(prop, result)
             self.assertIn(prop, result["user_settings"])
 
+    def test_pronouns_field_type_support(self) -> None:
+        hamlet = self.example_user("hamlet")
+        result = fetch_initial_state_data(
+            user_profile=hamlet,
+            pronouns_field_type_supported=False,
+        )
+        self.assertIn("custom_profile_fields", result)
+        custom_profile_fields = result["custom_profile_fields"]
+        pronouns_field = [field for field in custom_profile_fields if field["name"] == "Pronouns"][
+            0
+        ]
+        self.assertEqual(pronouns_field["type"], CustomProfileField.SHORT_TEXT)
+
+        result = fetch_initial_state_data(
+            user_profile=hamlet,
+            pronouns_field_type_supported=True,
+        )
+        self.assertIn("custom_profile_fields", result)
+        custom_profile_fields = result["custom_profile_fields"]
+        pronouns_field = [field for field in custom_profile_fields if field["name"] == "Pronouns"][
+            0
+        ]
+        self.assertEqual(pronouns_field["type"], CustomProfileField.PRONOUNS)
+
 
 class ClientDescriptorsTest(ZulipTestCase):
     def test_get_client_info_for_all_public_streams(self) -> None:
@@ -980,9 +1059,18 @@ class RestartEventsTest(ZulipTestCase):
         user_profile: UserProfile,
         post_data: Dict[str, Any],
         client_name: Optional[str] = None,
+        user_agent: Optional[str] = None,
     ) -> HttpResponse:
+        meta_data: Optional[Dict[str, Any]] = None
+        if user_agent is not None:
+            meta_data = {"HTTP_USER_AGENT": user_agent}
+
         request = HostRequestMock(
-            post_data, user_profile, client_name=client_name, tornado_handler=dummy_handler
+            post_data,
+            user_profile,
+            client_name=client_name,
+            tornado_handler=dummy_handler,
+            meta_data=meta_data,
         )
         return view_func(request, user_profile)
 
@@ -1099,6 +1187,7 @@ class RestartEventsTest(ZulipTestCase):
                     "dont_block": orjson.dumps(True).decode(),
                 },
                 client_name="website",
+                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
             )
 
 

zerver/tests/test_events.py

@@ -262,6 +262,7 @@ class BaseAction(ZulipTestCase):
         bulk_message_deletion: bool = True,
         stream_typing_notifications: bool = True,
         user_settings_object: bool = False,
+        pronouns_field_type_supported: bool = True,
     ) -> List[Dict[str, Any]]:
         """
         Make sure we have a clean slate of client descriptors for these tests.
@@ -289,6 +290,7 @@ class BaseAction(ZulipTestCase):
                 bulk_message_deletion=bulk_message_deletion,
                 stream_typing_notifications=stream_typing_notifications,
                 user_settings_object=user_settings_object,
+                pronouns_field_type_supported=pronouns_field_type_supported,
             )
         )
 
@@ -302,6 +304,7 @@ class BaseAction(ZulipTestCase):
             slim_presence=slim_presence,
             include_subscribers=include_subscribers,
             include_streams=include_streams,
+            pronouns_field_type_supported=pronouns_field_type_supported,
         )
 
         # We want even those `send_event` calls which have been hooked to
@@ -357,6 +360,7 @@ class BaseAction(ZulipTestCase):
             slim_presence=slim_presence,
             include_subscribers=include_subscribers,
             include_streams=include_streams,
+            pronouns_field_type_supported=pronouns_field_type_supported,
         )
         post_process_state(self.user_profile, normal_state, notification_settings_null)
         self.match_states(hybrid_state, normal_state, events)
@@ -997,6 +1001,33 @@ class NormalActionsTest(BaseAction):
         events = self.verify_action(lambda: do_remove_realm_custom_profile_field(realm, field))
         check_custom_profile_fields("events[0]", events[0])
 
+    def test_pronouns_type_support_in_custom_profile_fields_events(self) -> None:
+        realm = self.user_profile.realm
+        field = CustomProfileField.objects.get(realm=realm, name="Pronouns")
+        name = field.name
+        hint = "What pronouns should people use for you?"
+
+        events = self.verify_action(
+            lambda: try_update_realm_custom_profile_field(realm, field, name, hint=hint),
+            pronouns_field_type_supported=True,
+        )
+        check_custom_profile_fields("events[0]", events[0])
+        pronouns_field = [
+            field_obj for field_obj in events[0]["fields"] if field_obj["id"] == field.id
+        ][0]
+        self.assertEqual(pronouns_field["type"], CustomProfileField.PRONOUNS)
+
+        hint = "What pronouns should people use to refer you?"
+        events = self.verify_action(
+            lambda: try_update_realm_custom_profile_field(realm, field, name, hint=hint),
+            pronouns_field_type_supported=False,
+        )
+        check_custom_profile_fields("events[0]", events[0])
+        pronouns_field = [
+            field_obj for field_obj in events[0]["fields"] if field_obj["id"] == field.id
+        ][0]
+        self.assertEqual(pronouns_field["type"], CustomProfileField.SHORT_TEXT)
+
     def test_custom_profile_field_data_events(self) -> None:
         field_id = self.user_profile.realm.customprofilefield_set.get(
             realm=self.user_profile.realm, name="Biography"

zerver/tornado/django_api.py

@@ -75,6 +75,7 @@ def request_event_queue(
     bulk_message_deletion: bool = False,
     stream_typing_notifications: bool = False,
     user_settings_object: bool = False,
+    pronouns_field_type_supported: bool = True,
 ) -> Optional[str]:
 
     if not settings.USING_TORNADO:
@@ -96,6 +97,7 @@ def request_event_queue(
         "bulk_message_deletion": orjson.dumps(bulk_message_deletion),
         "stream_typing_notifications": orjson.dumps(stream_typing_notifications),
         "user_settings_object": orjson.dumps(user_settings_object),
+        "pronouns_field_type_supported": orjson.dumps(pronouns_field_type_supported),
     }
 
     if event_types is not None:

zerver/tornado/event_queue.py

@@ -44,6 +44,7 @@ from zerver.lib.notification_data import UserMessageNotificationsData
 from zerver.lib.queue import queue_json_publish, retry_event
 from zerver.lib.utils import statsd
 from zerver.middleware import async_request_timer_restart
+from zerver.models import CustomProfileField
 from zerver.tornado.descriptors import clear_descriptor_by_handler_id, set_descriptor_by_handler_id
 from zerver.tornado.exceptions import BadEventQueueIdError
 from zerver.tornado.handlers import (
@@ -94,6 +95,7 @@ class ClientDescriptor:
         bulk_message_deletion: bool = False,
         stream_typing_notifications: bool = False,
         user_settings_object: bool = False,
+        pronouns_field_type_supported: bool = True,
     ) -> None:
         # These objects are serialized on shutdown and restored on restart.
         # If fields are added or semantics are changed, temporary code must be
@@ -117,6 +119,7 @@ class ClientDescriptor:
         self.bulk_message_deletion = bulk_message_deletion
         self.stream_typing_notifications = stream_typing_notifications
         self.user_settings_object = user_settings_object
+        self.pronouns_field_type_supported = pronouns_field_type_supported
 
         # Default for lifespan_secs is DEFAULT_EVENT_QUEUE_TIMEOUT_SECS;
         # but users can set it as high as MAX_QUEUE_TIMEOUT_SECS.
@@ -144,6 +147,7 @@ class ClientDescriptor:
             bulk_message_deletion=self.bulk_message_deletion,
             stream_typing_notifications=self.stream_typing_notifications,
             user_settings_object=self.user_settings_object,
+            pronouns_field_type_supported=self.pronouns_field_type_supported,
         )
 
     def __repr__(self) -> str:
@@ -176,6 +180,7 @@ class ClientDescriptor:
             d.get("bulk_message_deletion", False),
             d.get("stream_typing_notifications", False),
             d.get("user_settings_object", False),
+            d.get("pronouns_field_type_supported", True),
         )
         ret.last_connection_time = d["last_connection_time"]
         return ret
@@ -1182,6 +1187,25 @@ def process_message_update_event(
                 client.add_event(user_event)
 
 
+def process_custom_profile_fields_event(event: Mapping[str, Any], users: Iterable[int]) -> None:
+    pronouns_type_unsupported_fields = copy.deepcopy(event["fields"])
+    for field in pronouns_type_unsupported_fields:
+        if field["type"] == CustomProfileField.PRONOUNS:
+            field["type"] = CustomProfileField.SHORT_TEXT
+
+    pronouns_type_unsupported_event = dict(
+        type="custom_profile_fields", fields=pronouns_type_unsupported_fields
+    )
+
+    for user_profile_id in users:
+        for client in get_client_descriptors_for_user(user_profile_id):
+            if client.accepts_event(event):
+                if not client.pronouns_field_type_supported:
+                    client.add_event(pronouns_type_unsupported_event)
+                    continue
+                client.add_event(event)
+
+
 def maybe_enqueue_notifications_for_message_update(
     user_notifications_data: UserMessageNotificationsData,
     message_id: int,
@@ -1314,6 +1338,8 @@ def process_notification(notice: Mapping[str, Any]) -> None:
         process_deletion_event(event, user_ids)
     elif event["type"] == "presence":
         process_presence_event(event, cast(List[int], users))
+    elif event["type"] == "custom_profile_fields":
+        process_custom_profile_fields_event(event, cast(List[int], users))
     else:
         process_event(event, cast(List[int], users))
     logging.debug(

zerver/tornado/views.py

@@ -116,6 +116,9 @@ def get_events_backend(
     user_settings_object: bool = REQ(
         default=False, json_validator=check_bool, intentionally_undocumented=True
     ),
+    pronouns_field_type_supported: bool = REQ(
+        default=True, json_validator=check_bool, intentionally_undocumented=True
+    ),
 ) -> HttpResponse:
     if all_public_streams and not user_profile.can_access_public_streams():
         raise JsonableError(_("User not authorized for this query"))
@@ -147,6 +150,7 @@ def get_events_backend(
             bulk_message_deletion=bulk_message_deletion,
             stream_typing_notifications=stream_typing_notifications,
             user_settings_object=user_settings_object,
+            pronouns_field_type_supported=pronouns_field_type_supported,
         )
 
     result = in_tornado_thread(fetch_events)(

zerver/views/events_register.py

@@ -6,6 +6,7 @@ from django.http import HttpRequest, HttpResponse
 from django.utils.translation import gettext as _
 
 from zerver.context_processors import get_valid_realm_from_request
+from zerver.lib.compatibility import is_pronouns_field_type_supported
 from zerver.lib.events import do_events_register
 from zerver.lib.exceptions import JsonableError, MissingAuthenticationError
 from zerver.lib.request import REQ, RequestNotes, has_request_variables
@@ -123,6 +124,7 @@ def events_register_backend(
     client = RequestNotes.get_notes(request).client
     assert client is not None
 
+    pronouns_field_type_supported = is_pronouns_field_type_supported(request.headers["User-Agent"])
     ret = do_events_register(
         user_profile,
         realm,
@@ -139,5 +141,6 @@ def events_register_backend(
         client_capabilities=client_capabilities,
         fetch_event_types=fetch_event_types,
         spectator_requested_language=spectator_requested_language,
+        pronouns_field_type_supported=pronouns_field_type_supported,
     )
     return json_success(request, data=ret)