presence: Add slim_presence flag.

This flag affects page_params and the
payload you get back from POSTs to this
url:

    users/me/presence

The flag does not yet affect the
presence events that get sent to a
client.

frontend_tests/node_tests/presence.js

@@ -151,21 +151,21 @@ run_test('set_presence_info', () => {
     const presences = {};
     const base_time = 500;
 
-    presences[alice.email] = {
+    presences[alice.user_id.toString()] = {
         website: {
             status: 'active',
             timestamp: base_time,
         },
     };
 
-    presences[fred.email] = {
+    presences[fred.user_id.toString()] = {
         website: {
             status: 'idle',
             timestamp: base_time,
         },
     };
 
-    presences[me.email] = {
+    presences[me.user_id.toString()] = {
         website: {
             status: 'active',
             timestamp: base_time,
@@ -197,19 +197,6 @@ run_test('set_presence_info', () => {
     people.get_realm_count = function () { return 1000; };
     assert.equal(presence.set_info(presences, base_time), undefined);
     people.get_realm_count = get_realm_count;
-
-    const unknown = {
-        email: 'unknown@zulip.com',
-        user_id: 42,
-        full_name: 'Unknown Name',
-    };
-    presences[unknown.email] = {};
-
-    server_events.suspect_offline = false;
-    blueslip.error = function (msg) {
-        assert.equal(msg, 'Unknown email in presence data: unknown@zulip.com');
-    };
-    presence.set_info(presences, base_time);
 });
 
 run_test('last_active_date', () => {

static/js/activity.js

@@ -305,6 +305,7 @@ function send_presence_to_server(want_redraw) {
             status: exports.compute_active_status(),
             ping_only: !want_redraw,
             new_user_input: exports.new_user_input,
+            slim_presence: true,
         },
         idempotent: true,
         success: function (data) {

static/js/presence.js

@@ -96,27 +96,12 @@ exports.set_info_for_user = function (user_id, info, server_time) {
 
 exports.set_info = function (presences, server_timestamp) {
     exports.presence_info = {};
-    _.each(presences, function (info, this_email) {
+    _.each(presences, function (info, user_id_str) {
-        const person = people.get_by_email(this_email);
+        const status = status_from_timestamp(server_timestamp,
+                                             info);
 
-        if (person === undefined) {
+        const user_id = parseInt(user_id_str, 10);
-            if (!(server_events.suspect_offline || reload_state.is_in_progress())) {
+        exports.presence_info[user_id] = status;
-                // If we're online, and we get a user who we don't
-                // know about in the presence data, throw an error.
-                blueslip.error('Unknown email in presence data: ' + this_email);
-            }
-            // Either way, we deal by skipping this user and
-            // rendering everyone else, to avoid disruption.
-            return;
-        }
-
-        const user_id = person.user_id;
-
-        if (user_id) {
-            const status = status_from_timestamp(server_timestamp,
-                                                 info);
-            exports.presence_info[user_id] = status;
-        }
     });
     exports.update_info_for_small_realm();
 };

static/js/server_events.js

@@ -198,6 +198,7 @@ function get_events(options) {
     }
 
     get_events_params.client_gravatar = true;
+    get_events_params.slim_presence = true;
 
     get_events_timeout = undefined;
     get_events_xhr = channel.get({

zerver/lib/actions.py

@@ -4959,12 +4959,14 @@ def filter_presence_idle_user_ids(user_ids: Set[int]) -> List[int]:
     idle_user_ids = user_ids - active_user_ids
     return sorted(list(idle_user_ids))
 
-def get_status_dict(requesting_user_profile: UserProfile) -> Dict[str, Dict[str, Dict[str, Any]]]:
+def get_status_dict(requesting_user_profile: UserProfile,
+                    slim_presence: bool) -> Dict[str, Dict[str, Dict[str, Any]]]:
+
     if requesting_user_profile.realm.presence_disabled:
         # Return an empty dict if presence is disabled in this realm
         return defaultdict(dict)
 
-    return UserPresence.get_status_dict_by_realm(requesting_user_profile.realm_id)
+    return UserPresence.get_status_dict_by_realm(requesting_user_profile.realm_id, slim_presence)
 
 def do_send_confirmation_email(invitee: PreregistrationUser,
                                referrer: UserProfile) -> str:

zerver/lib/events.py

@@ -80,6 +80,7 @@ def always_want(msg_type: str) -> bool:
 def fetch_initial_state_data(user_profile: UserProfile,
                              event_types: Optional[Iterable[str]],
                              queue_id: str, client_gravatar: bool,
+                             slim_presence: bool = False,
                              include_subscribers: bool = True) -> Dict[str, Any]:
     state = {'queue_id': queue_id}  # type: Dict[str, Any]
     realm = user_profile.realm
@@ -118,7 +119,7 @@ def fetch_initial_state_data(user_profile: UserProfile,
         state['pointer'] = user_profile.pointer
 
     if want('presence'):
-        state['presences'] = get_status_dict(user_profile)
+        state['presences'] = get_status_dict(user_profile, slim_presence)
 
     if want('realm'):
         for property_name in Realm.property_types:
@@ -310,7 +311,7 @@ def fetch_initial_state_data(user_profile: UserProfile,
 
 def apply_events(state: Dict[str, Any], events: Iterable[Dict[str, Any]],
                  user_profile: UserProfile, client_gravatar: bool,
-                 include_subscribers: bool = True,
+                 slim_presence: bool, include_subscribers: bool = True,
                  fetch_event_types: Optional[Iterable[str]] = None) -> None:
     for event in events:
         if fetch_event_types is not None and event['type'] not in fetch_event_types:
@@ -323,12 +324,14 @@ def apply_events(state: Dict[str, Any], events: Iterable[Dict[str, Any]],
             # `apply_event`.  For now, be careful in your choice of
             # `fetch_event_types`.
             continue
-        apply_event(state, event, user_profile, client_gravatar, include_subscribers)
+        apply_event(state, event, user_profile,
+                    client_gravatar, slim_presence, include_subscribers)
 
 def apply_event(state: Dict[str, Any],
                 event: Dict[str, Any],
                 user_profile: UserProfile,
                 client_gravatar: bool,
+                slim_presence: bool,
                 include_subscribers: bool) -> None:
     if event['type'] == "message":
         state['max_message_id'] = max(state['max_message_id'], event['message']['id'])
@@ -605,7 +608,11 @@ def apply_event(state: Dict[str, Any],
     elif event['type'] == "presence":
         # TODO: Add user_id to presence update events / state format!
         presence_user_profile = get_user(event['email'], user_profile.realm)
-        state['presences'][event['email']] = UserPresence.get_status_dict_by_user(
+        if slim_presence:
+            user_key = str(presence_user_profile.id)
+        else:
+            user_key = event['email']
+        state['presences'][user_key] = UserPresence.get_status_dict_by_user(
             presence_user_profile)[event['email']]
     elif event['type'] == "update_message":
         # We don't return messages in /register, so we don't need to
@@ -761,6 +768,7 @@ def apply_event(state: Dict[str, Any],
 def do_events_register(user_profile: UserProfile, user_client: Client,
                        apply_markdown: bool = True,
                        client_gravatar: bool = False,
+                       slim_presence: bool = False,
                        event_types: Optional[Iterable[str]] = None,
                        queue_lifespan_secs: int = 0,
                        all_public_streams: bool = False,
@@ -780,7 +788,8 @@ def do_events_register(user_profile: UserProfile, user_client: Client,
 
     # Note that we pass event_types, not fetch_event_types here, since
     # that's what controls which future events are sent.
-    queue_id = request_event_queue(user_profile, user_client, apply_markdown, client_gravatar,
+    queue_id = request_event_queue(user_profile, user_client,
+                                   apply_markdown, client_gravatar, slim_presence,
                                    queue_lifespan_secs, event_types, all_public_streams,
                                    narrow=narrow)
 
@@ -799,12 +808,13 @@ def do_events_register(user_profile: UserProfile, user_client: Client,
 
     ret = fetch_initial_state_data(user_profile, event_types_set, queue_id,
                                    client_gravatar=client_gravatar,
+                                   slim_presence=slim_presence,
                                    include_subscribers=include_subscribers)
 
     # Apply events that came in while we were fetching initial data
     events = get_user_events(user_profile, queue_id, -1)
     apply_events(ret, events, user_profile, include_subscribers=include_subscribers,
-                 client_gravatar=client_gravatar,
+                 client_gravatar=client_gravatar, slim_presence=slim_presence,
                  fetch_event_types=fetch_event_types)
 
     post_process_state(user_profile, ret, notification_settings_null)

zerver/openapi/zulip.yaml

@@ -2035,6 +2035,14 @@ paths:
           type: boolean
           default: false
         example: true
+      - name: slim_presence
+        in: query
+        description: Setting this to `true` will make presence dictionaries be
+          keyed by user_id instead of email.
+        schema:
+          type: boolean
+          default: false
+        example: true
       - name: event_types
         in: query
         description: "A JSON-encoded array indicating which types of events

zerver/tests/test_events.py

@@ -484,7 +484,8 @@ class EventsRegisterTest(ZulipTestCase):
     def do_test(self, action: Callable[[], object], event_types: Optional[List[str]]=None,
                 include_subscribers: bool=True, state_change_expected: bool=True,
                 notification_settings_null: bool=False,
-                client_gravatar: bool=True, num_events: int=1) -> List[Dict[str, Any]]:
+                client_gravatar: bool=True, slim_presence: bool=False,
+                num_events: int=1) -> List[Dict[str, Any]]:
         '''
         Make sure we have a clean slate of client descriptors for these tests.
         If we don't do this, then certain failures will only manifest when you
@@ -502,6 +503,7 @@ class EventsRegisterTest(ZulipTestCase):
                  client_type_name = "website",
                  apply_markdown = True,
                  client_gravatar = client_gravatar,
+                 slim_presence = slim_presence,
                  all_public_streams = False,
                  queue_timeout = 600,
                  last_connection_time = time.time(),
@@ -512,6 +514,7 @@ class EventsRegisterTest(ZulipTestCase):
         hybrid_state = fetch_initial_state_data(
             self.user_profile, event_types, "",
             client_gravatar=client_gravatar,
+            slim_presence=slim_presence,
             include_subscribers=include_subscribers
         )
         action()
@@ -522,7 +525,9 @@ class EventsRegisterTest(ZulipTestCase):
         post_process_state(self.user_profile, initial_state, notification_settings_null)
         before = ujson.dumps(initial_state)
         apply_events(hybrid_state, events, self.user_profile,
-                     client_gravatar=client_gravatar, include_subscribers=include_subscribers)
+                     client_gravatar=client_gravatar,
+                     slim_presence=slim_presence,
+                     include_subscribers=include_subscribers)
         post_process_state(self.user_profile, hybrid_state, notification_settings_null)
         after = ujson.dumps(hybrid_state)
 
@@ -540,6 +545,7 @@ class EventsRegisterTest(ZulipTestCase):
         normal_state = fetch_initial_state_data(
             self.user_profile, event_types, "",
             client_gravatar=client_gravatar,
+            slim_presence=slim_presence,
             include_subscribers=include_subscribers,
         )
         post_process_state(self.user_profile, normal_state, notification_settings_null)
@@ -1194,8 +1200,18 @@ class EventsRegisterTest(ZulipTestCase):
                 ])),
             ])),
         ])
+
         events = self.do_test(lambda: do_update_user_presence(
-            self.user_profile, get_client("website"), timezone_now(), UserPresence.ACTIVE))
+            self.user_profile, get_client("website"),
+            timezone_now(), UserPresence.ACTIVE),
+            slim_presence=False)
+        error = schema_checker('events[0]', events[0])
+        self.assert_on_error(error)
+
+        events = self.do_test(lambda: do_update_user_presence(
+            self.example_user('cordelia'), get_client("website"),
+            timezone_now(), UserPresence.ACTIVE),
+            slim_presence=True)
         error = schema_checker('events[0]', events[0])
         self.assert_on_error(error)
 

zerver/tests/test_presence.py

@@ -501,17 +501,29 @@ class UserPresenceAggregationTests(ZulipTestCase):
 class GetRealmStatusesTest(ZulipTestCase):
     def test_get_statuses(self) -> None:
         # Setup the test by simulating users reporting their presence data.
-        othello_email = self.example_email("othello")
+        othello = self.example_user("othello")
-        result = self.api_post(othello_email, "/api/v1/users/me/presence", {'status': 'active'},
+        hamlet = self.example_user("hamlet")
+
+        result = self.api_post(othello.email, "/api/v1/users/me/presence",
+                               dict(status='active'),
                                HTTP_USER_AGENT="ZulipAndroid/1.0")
 
-        hamlet_email = self.example_email("hamlet")
+        result = self.api_post(hamlet.email, "/api/v1/users/me/presence",
-        result = self.api_post(hamlet_email, "/api/v1/users/me/presence", {'status': 'idle'},
+                               dict(status='idle'),
                                HTTP_USER_AGENT="ZulipDesktop/1.0")
         self.assert_json_success(result)
+        json = result.json()
+        self.assertEqual(set(json['presences'].keys()), {hamlet.email, othello.email})
+
+        result = self.api_post(hamlet.email, "/api/v1/users/me/presence",
+                               dict(status='active', slim_presence='true'),
+                               HTTP_USER_AGENT="ZulipDesktop/1.0")
+        self.assert_json_success(result)
+        json = result.json()
+        self.assertEqual(set(json['presences'].keys()), {str(hamlet.id), str(othello.id)})
 
         # Check that a bot can fetch the presence data for the realm.
         result = self.api_get(self.example_email("default_bot"), "/api/v1/realm/presence")
         self.assert_json_success(result)
         json = result.json()
-        self.assertEqual(sorted(json['presences'].keys()), [hamlet_email, othello_email])
+        self.assertEqual(set(json['presences'].keys()), {hamlet.email, othello.email})

zerver/tornado/event_queue.py

@@ -68,6 +68,7 @@ class ClientDescriptor:
                  client_type_name: str,
                  apply_markdown: bool=True,
                  client_gravatar: bool=True,
+                 slim_presence: bool=False,
                  all_public_streams: bool=False,
                  lifespan_secs: int=0,
                  narrow: Iterable[Sequence[str]]=[]) -> None:
@@ -84,6 +85,7 @@ class ClientDescriptor:
         self.last_connection_time = time.time()
         self.apply_markdown = apply_markdown
         self.client_gravatar = client_gravatar
+        self.slim_presence = slim_presence
         self.all_public_streams = all_public_streams
         self.client_type_name = client_type_name
         self._timeout_handle = None  # type: Any # TODO: should be return type of ioloop.call_later
@@ -108,6 +110,7 @@ class ClientDescriptor:
                     last_connection_time=self.last_connection_time,
                     apply_markdown=self.apply_markdown,
                     client_gravatar=self.client_gravatar,
+                    slim_presence=self.slim_presence,
                     all_public_streams=self.all_public_streams,
                     narrow=self.narrow,
                     client_type_name=self.client_type_name)
@@ -124,6 +127,9 @@ class ClientDescriptor:
             # Temporary migration for the addition of the client_gravatar field
             d['client_gravatar'] = False
 
+        if 'slim_presence' not in d:
+            d['slim_presence'] = False
+
         ret = cls(
             d['user_profile_id'],
             d['realm_id'],
@@ -132,6 +138,7 @@ class ClientDescriptor:
             d['client_type_name'],
             d['apply_markdown'],
             d['client_gravatar'],
+            d['slim_presence'],
             d['all_public_streams'],
             d['queue_timeout'],
             d.get('narrow', [])
@@ -570,7 +577,7 @@ def fetch_events(query: Mapping[str, Any]) -> Dict[str, Any]:
 # The following functions are called from Django
 
 def request_event_queue(user_profile: UserProfile, user_client: Client, apply_markdown: bool,
-                        client_gravatar: bool, queue_lifespan_secs: int,
+                        client_gravatar: bool, slim_presence: bool, queue_lifespan_secs: int,
                         event_types: Optional[Iterable[str]]=None,
                         all_public_streams: bool=False,
                         narrow: Iterable[Sequence[str]]=[]) -> Optional[str]:
@@ -579,6 +586,7 @@ def request_event_queue(user_profile: UserProfile, user_client: Client, apply_ma
         req = {'dont_block': 'true',
                'apply_markdown': ujson.dumps(apply_markdown),
                'client_gravatar': ujson.dumps(client_gravatar),
+               'slim_presence': ujson.dumps(slim_presence),
                'all_public_streams': ujson.dumps(all_public_streams),
                'client': 'internal',
                'user_profile_id': user_profile.id,

zerver/tornado/views.py

@@ -67,6 +67,8 @@ def get_events_backend(request: HttpRequest, user_profile: UserProfile, handler:
                                                 intentionally_undocumented=True),
                        client_gravatar: bool=REQ(default=False, validator=check_bool,
                                                  intentionally_undocumented=True),
+                       slim_presence: bool=REQ(default=False, validator=check_bool,
+                                               intentionally_undocumented=True),
                        all_public_streams: bool=REQ(default=False, validator=check_bool,
                                                     intentionally_undocumented=True),
                        event_types: Optional[str]=REQ(default=None, validator=check_list(check_string),
@@ -102,6 +104,7 @@ def get_events_backend(request: HttpRequest, user_profile: UserProfile, handler:
             client_type_name = valid_user_client.name,
             apply_markdown = apply_markdown,
             client_gravatar = client_gravatar,
+            slim_presence = slim_presence,
             all_public_streams = all_public_streams,
             queue_timeout = lifespan_secs,
             last_connection_time = time.time(),

zerver/views/events_register.py

@@ -27,6 +27,7 @@ def events_register_backend(
         request: HttpRequest, user_profile: UserProfile,
         apply_markdown: bool=REQ(default=False, validator=check_bool),
         client_gravatar: bool=REQ(default=False, validator=check_bool),
+        slim_presence: bool=REQ(default=False, validator=check_bool),
         all_public_streams: Optional[bool]=REQ(default=None, validator=check_bool),
         include_subscribers: bool=REQ(default=False, validator=check_bool),
         client_capabilities: Optional[Dict[str, bool]]=REQ(validator=check_dict([
@@ -49,7 +50,8 @@ def events_register_backend(
         client_capabilities = {}
     notification_settings_null = client_capabilities.get("notification_settings_null", False)
 
-    ret = do_events_register(user_profile, request.client, apply_markdown, client_gravatar,
+    ret = do_events_register(user_profile, request.client,
+                             apply_markdown, client_gravatar, slim_presence,
                              event_types, queue_lifespan_secs, all_public_streams,
                              narrow=narrow, include_subscribers=include_subscribers,
                              notification_settings_null=notification_settings_null,

zerver/views/home.py

@@ -121,6 +121,7 @@ def home_real(request: HttpRequest) -> HttpResponse:
 
     register_ret = do_events_register(user_profile, request.client,
                                       apply_markdown=True, client_gravatar=True,
+                                      slim_presence=True,
                                       notification_settings_null=True,
                                       narrow=narrow)
     user_has_messages = (register_ret['max_message_id'] != -1)

zerver/views/presence.py

@@ -21,12 +21,16 @@ from zerver.lib.validator import check_bool, check_capped_string
 from zerver.models import UserActivity, UserPresence, UserProfile, \
     get_active_user_by_delivery_email
 
-def get_status_list(requesting_user_profile: UserProfile) -> Dict[str, Any]:
+def get_status_list(requesting_user_profile: UserProfile,
-    return {'presences': get_status_dict(requesting_user_profile),
+                    slim_presence: bool) -> Dict[str, Any]:
+    return {'presences': get_status_dict(requesting_user_profile, slim_presence),
             'server_timestamp': time.time()}
 
 def get_presence_backend(request: HttpRequest, user_profile: UserProfile,
                          email: str) -> HttpResponse:
+    # This isn't used by the webapp; it's available for API use by
+    # bots and other clients.  We may want to add slim_presence
+    # support for it (or just migrate its API wholesale) later.
     try:
         target = get_active_user_by_delivery_email(email, user_profile.realm)
     except UserProfile.DoesNotExist:
@@ -78,7 +82,8 @@ def update_user_status_backend(request: HttpRequest,
 def update_active_status_backend(request: HttpRequest, user_profile: UserProfile,
                                  status: str=REQ(),
                                  ping_only: bool=REQ(validator=check_bool, default=False),
-                                 new_user_input: bool=REQ(validator=check_bool, default=False)
+                                 new_user_input: bool=REQ(validator=check_bool, default=False),
+                                 slim_presence: bool=REQ(validator=check_bool, default=False)
                                  ) -> HttpResponse:
     status_val = UserPresence.status_from_string(status)
     if status_val is None:
@@ -90,7 +95,7 @@ def update_active_status_backend(request: HttpRequest, user_profile: UserProfile
     if ping_only:
         ret = {}  # type: Dict[str, Any]
     else:
-        ret = get_status_list(user_profile)
+        ret = get_status_list(user_profile, slim_presence)
 
     if user_profile.realm.is_zephyr_mirror_realm:
         # In zephyr mirroring realms, users can't see the presence of other
@@ -109,4 +114,7 @@ def update_active_status_backend(request: HttpRequest, user_profile: UserProfile
     return json_success(ret)
 
 def get_statuses_for_realm(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
-    return json_success(get_status_list(user_profile))
+    # This isn't used by the webapp; it's available for API use by
+    # bots and other clients.  We may want to add slim_presence
+    # support for it (or just migrate its API wholesale) later.
+    return json_success(get_status_list(user_profile, slim_presence=False))