diff --git a/zerver/actions/presence.py b/zerver/actions/presence.py new file mode 100644 index 0000000000..19e0943e84 --- /dev/null +++ b/zerver/actions/presence.py @@ -0,0 +1,173 @@ +import datetime +import time +from typing import Optional + +from django.conf import settings + +from zerver.actions.user_activity import update_user_activity_interval +from zerver.decorator import statsd_increment +from zerver.lib.queue import queue_json_publish +from zerver.lib.timestamp import datetime_to_timestamp +from zerver.lib.user_status import update_user_status +from zerver.models import Client, UserPresence, UserProfile, UserStatus, active_user_ids, get_client +from zerver.tornado.django_api import send_event + + +def send_presence_changed(user_profile: UserProfile, presence: UserPresence) -> None: + # Most presence data is sent to clients in the main presence + # endpoint in response to the user's own presence; this results + # data that is 1-2 minutes stale for who is online. The flaw with + # this plan is when a user comes back online and then immediately + # sends a message, recipients may still see that user as offline! + # We solve that by sending an immediate presence update clients. + # + # See https://zulip.readthedocs.io/en/latest/subsystems/presence.html for + # internals documentation on presence. + user_ids = active_user_ids(user_profile.realm_id) + if len(user_ids) > settings.USER_LIMIT_FOR_SENDING_PRESENCE_UPDATE_EVENTS: + # These immediate presence generate quadratic work for Tornado + # (linear number of users in each event and the frequency of + # users coming online grows linearly with userbase too). In + # organizations with thousands of users, this can overload + # Tornado, especially if much of the realm comes online at the + # same time. + # + # The utility of these live-presence updates goes down as + # organizations get bigger (since one is much less likely to + # be paying attention to the sidebar); so beyond a limit, we + # stop sending them at all. + return + + presence_dict = presence.to_dict() + event = dict( + type="presence", + email=user_profile.email, + user_id=user_profile.id, + server_timestamp=time.time(), + presence={presence_dict["client"]: presence_dict}, + ) + send_event(user_profile.realm, event, user_ids) + + +def consolidate_client(client: Client) -> Client: + # The web app reports a client as 'website' + # The desktop app reports a client as ZulipDesktop + # due to it setting a custom user agent. We want both + # to count as web users + + # Alias ZulipDesktop to website + if client.name in ["ZulipDesktop"]: + return get_client("website") + else: + return client + + +@statsd_increment("user_presence") +def do_update_user_presence( + user_profile: UserProfile, client: Client, log_time: datetime.datetime, status: int +) -> None: + client = consolidate_client(client) + + defaults = dict( + timestamp=log_time, + status=status, + realm_id=user_profile.realm_id, + ) + + (presence, created) = UserPresence.objects.get_or_create( + user_profile=user_profile, + client=client, + defaults=defaults, + ) + + stale_status = (log_time - presence.timestamp) > datetime.timedelta(minutes=1, seconds=10) + was_idle = presence.status == UserPresence.IDLE + became_online = (status == UserPresence.ACTIVE) and (stale_status or was_idle) + + # If an object was created, it has already been saved. + # + # We suppress changes from ACTIVE to IDLE before stale_status is reached; + # this protects us from the user having two clients open: one active, the + # other idle. Without this check, we would constantly toggle their status + # between the two states. + if not created and stale_status or was_idle or status == presence.status: + # The following block attempts to only update the "status" + # field in the event that it actually changed. This is + # important to avoid flushing the UserPresence cache when the + # data it would return to a client hasn't actually changed + # (see the UserPresence post_save hook for details). + presence.timestamp = log_time + update_fields = ["timestamp"] + if presence.status != status: + presence.status = status + update_fields.append("status") + presence.save(update_fields=update_fields) + + if not user_profile.realm.presence_disabled and (created or became_online): + send_presence_changed(user_profile, presence) + + +def update_user_presence( + user_profile: UserProfile, + client: Client, + log_time: datetime.datetime, + status: int, + new_user_input: bool, +) -> None: + event = { + "user_profile_id": user_profile.id, + "status": status, + "time": datetime_to_timestamp(log_time), + "client": client.name, + } + + queue_json_publish("user_presence", event) + + if new_user_input: + update_user_activity_interval(user_profile, log_time) + + +def do_update_user_status( + user_profile: UserProfile, + away: Optional[bool], + status_text: Optional[str], + client_id: int, + emoji_name: Optional[str], + emoji_code: Optional[str], + reaction_type: Optional[str], +) -> None: + if away is None: + status = None + elif away: + status = UserStatus.AWAY + else: + status = UserStatus.NORMAL + + realm = user_profile.realm + + update_user_status( + user_profile_id=user_profile.id, + status=status, + status_text=status_text, + client_id=client_id, + emoji_name=emoji_name, + emoji_code=emoji_code, + reaction_type=reaction_type, + ) + + event = dict( + type="user_status", + user_id=user_profile.id, + ) + + if away is not None: + event["away"] = away + + if status_text is not None: + event["status_text"] = status_text + + if emoji_name is not None: + event["emoji_name"] = emoji_name + event["emoji_code"] = emoji_code + event["reaction_type"] = reaction_type + send_event(realm, event, active_user_ids(realm.id)) diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index f6944b06be..9d20d0836f 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -2,7 +2,6 @@ import datetime import hashlib import itertools import logging -import time from collections import defaultdict from dataclasses import asdict, dataclass, field from operator import itemgetter @@ -46,13 +45,11 @@ from zerver.actions.default_streams import ( ) from zerver.actions.invites import notify_invites_changed, revoke_invites_generated_by_user from zerver.actions.uploads import check_attachment_reference_change, do_claim_attachments -from zerver.actions.user_activity import update_user_activity_interval from zerver.actions.user_groups import ( do_send_user_group_members_update_event, update_users_in_full_members_system_group, ) from zerver.actions.user_topics import do_mute_topic, do_unmute_topic -from zerver.decorator import statsd_increment from zerver.lib import retention as retention from zerver.lib.addressee import Addressee from zerver.lib.alert_words import get_alert_word_automaton @@ -181,7 +178,6 @@ from zerver.lib.user_groups import ( ) from zerver.lib.user_message import UserMessageLite, bulk_insert_ums from zerver.lib.user_mutes import add_user_mute, get_muting_users, get_user_mutes -from zerver.lib.user_status import update_user_status from zerver.lib.user_topics import get_users_muting_topic, remove_topic_mute from zerver.lib.users import ( check_bot_name_available, @@ -224,7 +220,6 @@ from zerver.models import ( UserMessage, UserPresence, UserProfile, - UserStatus, active_non_guest_user_ids, active_user_ids, bot_owner_user_ids, @@ -5370,166 +5365,6 @@ def get_default_subs(user_profile: UserProfile) -> List[Stream]: return get_default_streams_for_realm(user_profile.realm_id) -def send_presence_changed(user_profile: UserProfile, presence: UserPresence) -> None: - # Most presence data is sent to clients in the main presence - # endpoint in response to the user's own presence; this results - # data that is 1-2 minutes stale for who is online. The flaw with - # this plan is when a user comes back online and then immediately - # sends a message, recipients may still see that user as offline! - # We solve that by sending an immediate presence update clients. - # - # See https://zulip.readthedocs.io/en/latest/subsystems/presence.html for - # internals documentation on presence. - user_ids = active_user_ids(user_profile.realm_id) - if len(user_ids) > settings.USER_LIMIT_FOR_SENDING_PRESENCE_UPDATE_EVENTS: - # These immediate presence generate quadratic work for Tornado - # (linear number of users in each event and the frequency of - # users coming online grows linearly with userbase too). In - # organizations with thousands of users, this can overload - # Tornado, especially if much of the realm comes online at the - # same time. - # - # The utility of these live-presence updates goes down as - # organizations get bigger (since one is much less likely to - # be paying attention to the sidebar); so beyond a limit, we - # stop sending them at all. - return - - presence_dict = presence.to_dict() - event = dict( - type="presence", - email=user_profile.email, - user_id=user_profile.id, - server_timestamp=time.time(), - presence={presence_dict["client"]: presence_dict}, - ) - send_event(user_profile.realm, event, user_ids) - - -def consolidate_client(client: Client) -> Client: - # The web app reports a client as 'website' - # The desktop app reports a client as ZulipDesktop - # due to it setting a custom user agent. We want both - # to count as web users - - # Alias ZulipDesktop to website - if client.name in ["ZulipDesktop"]: - return get_client("website") - else: - return client - - -@statsd_increment("user_presence") -def do_update_user_presence( - user_profile: UserProfile, client: Client, log_time: datetime.datetime, status: int -) -> None: - client = consolidate_client(client) - - defaults = dict( - timestamp=log_time, - status=status, - realm_id=user_profile.realm_id, - ) - - (presence, created) = UserPresence.objects.get_or_create( - user_profile=user_profile, - client=client, - defaults=defaults, - ) - - stale_status = (log_time - presence.timestamp) > datetime.timedelta(minutes=1, seconds=10) - was_idle = presence.status == UserPresence.IDLE - became_online = (status == UserPresence.ACTIVE) and (stale_status or was_idle) - - # If an object was created, it has already been saved. - # - # We suppress changes from ACTIVE to IDLE before stale_status is reached; - # this protects us from the user having two clients open: one active, the - # other idle. Without this check, we would constantly toggle their status - # between the two states. - if not created and stale_status or was_idle or status == presence.status: - # The following block attempts to only update the "status" - # field in the event that it actually changed. This is - # important to avoid flushing the UserPresence cache when the - # data it would return to a client hasn't actually changed - # (see the UserPresence post_save hook for details). - presence.timestamp = log_time - update_fields = ["timestamp"] - if presence.status != status: - presence.status = status - update_fields.append("status") - presence.save(update_fields=update_fields) - - if not user_profile.realm.presence_disabled and (created or became_online): - send_presence_changed(user_profile, presence) - - -def update_user_presence( - user_profile: UserProfile, - client: Client, - log_time: datetime.datetime, - status: int, - new_user_input: bool, -) -> None: - event = { - "user_profile_id": user_profile.id, - "status": status, - "time": datetime_to_timestamp(log_time), - "client": client.name, - } - - queue_json_publish("user_presence", event) - - if new_user_input: - update_user_activity_interval(user_profile, log_time) - - -def do_update_user_status( - user_profile: UserProfile, - away: Optional[bool], - status_text: Optional[str], - client_id: int, - emoji_name: Optional[str], - emoji_code: Optional[str], - reaction_type: Optional[str], -) -> None: - if away is None: - status = None - elif away: - status = UserStatus.AWAY - else: - status = UserStatus.NORMAL - - realm = user_profile.realm - - update_user_status( - user_profile_id=user_profile.id, - status=status, - status_text=status_text, - client_id=client_id, - emoji_name=emoji_name, - emoji_code=emoji_code, - reaction_type=reaction_type, - ) - - event = dict( - type="user_status", - user_id=user_profile.id, - ) - - if away is not None: - event["away"] = away - - if status_text is not None: - event["status_text"] = status_text - - if emoji_name is not None: - event["emoji_name"] = emoji_name - event["emoji_code"] = emoji_code - event["reaction_type"] = reaction_type - send_event(realm, event, active_user_ids(realm.id)) - - @dataclass class ReadMessagesEvent: messages: List[int] diff --git a/zerver/openapi/curl_param_value_generators.py b/zerver/openapi/curl_param_value_generators.py index 8bfdbc4c3b..ac18d750fd 100644 --- a/zerver/openapi/curl_param_value_generators.py +++ b/zerver/openapi/curl_param_value_generators.py @@ -10,9 +10,10 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple from django.utils.timezone import now as timezone_now +from zerver.actions.presence import update_user_presence from zerver.actions.realm_linkifiers import do_add_linkifier from zerver.actions.realm_playgrounds import do_add_realm_playground -from zerver.lib.actions import do_add_reaction, do_create_user, update_user_presence +from zerver.lib.actions import do_add_reaction, do_create_user from zerver.lib.events import do_events_register from zerver.lib.initial_password import initial_password from zerver.lib.test_classes import ZulipTestCase diff --git a/zerver/tests/test_event_system.py b/zerver/tests/test_event_system.py index 6939b78ac4..479aeaa5d5 100644 --- a/zerver/tests/test_event_system.py +++ b/zerver/tests/test_event_system.py @@ -8,12 +8,8 @@ from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from version import API_FEATURE_LEVEL, ZULIP_MERGE_BASE, ZULIP_VERSION -from zerver.lib.actions import ( - check_send_message, - do_change_user_role, - do_set_realm_property, - do_update_user_presence, -) +from zerver.actions.presence import do_update_user_presence +from zerver.lib.actions import check_send_message, do_change_user_role, do_set_realm_property from zerver.lib.event_schema import check_restart_event from zerver.lib.events import fetch_initial_state_data from zerver.lib.exceptions import AccessDeniedError diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 593ae991c3..99dcb5a6f7 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -32,6 +32,7 @@ from zerver.actions.invites import ( do_revoke_multi_use_invite, do_revoke_user_invite, ) +from zerver.actions.presence import do_update_user_presence, do_update_user_status from zerver.actions.realm_emoji import check_add_realm_emoji, do_remove_realm_emoji from zerver.actions.realm_icon import do_change_icon_source from zerver.actions.realm_linkifiers import ( @@ -99,8 +100,6 @@ from zerver.lib.actions import ( do_update_message_flags, do_update_outgoing_webhook_service, do_update_user_custom_profile_data_if_changed, - do_update_user_presence, - do_update_user_status, try_add_realm_custom_profile_field, try_update_realm_custom_profile_field, ) diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 85357fd989..c90dd66d37 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -11,6 +11,7 @@ from django.utils.timezone import now as timezone_now from analytics.models import UserCount from zerver.actions.alert_words import do_add_alert_words +from zerver.actions.presence import do_update_user_presence, do_update_user_status from zerver.actions.realm_emoji import check_add_realm_emoji from zerver.actions.realm_icon import do_change_icon_source from zerver.actions.realm_logo import do_change_logo_source @@ -25,8 +26,6 @@ from zerver.lib.actions import ( do_deactivate_user, do_mute_user, do_update_user_custom_profile_data_if_changed, - do_update_user_presence, - do_update_user_status, try_add_realm_custom_profile_field, ) from zerver.lib.avatar_hash import user_avatar_path diff --git a/zerver/views/presence.py b/zerver/views/presence.py index 869c259bca..e552a9e0f7 100644 --- a/zerver/views/presence.py +++ b/zerver/views/presence.py @@ -6,8 +6,8 @@ from django.http import HttpRequest, HttpResponse from django.utils.timezone import now as timezone_now from django.utils.translation import gettext as _ +from zerver.actions.presence import do_update_user_status, update_user_presence from zerver.decorator import human_users_only -from zerver.lib.actions import do_update_user_status, update_user_presence from zerver.lib.emoji import check_emoji_request, emoji_name_to_emoji_code from zerver.lib.exceptions import JsonableError from zerver.lib.presence import get_presence_for_user, get_presence_response diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index ae69ef3afe..9efd781dfd 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -47,13 +47,13 @@ from sentry_sdk import add_breadcrumb, configure_scope from zulip_bots.lib import extract_query_without_mention from zerver.actions.invites import do_send_confirmation_email +from zerver.actions.presence import do_update_user_presence from zerver.actions.realm_export import notify_realm_export from zerver.actions.user_activity import do_update_user_activity, do_update_user_activity_interval from zerver.context_processors import common_context from zerver.lib.actions import ( do_mark_stream_messages_as_read, do_update_embedded_data, - do_update_user_presence, internal_send_private_message, render_incoming_message, )