stream: Add field to track active status of stream.

api_docs/changelog.md

@@ -20,6 +20,14 @@ format used by the Zulip server that they are interacting with.
 
 ## Changes in Zulip 10.0
 
+**Feature level 323**
+
+* [`POST /register`](/api/register-queue), [`GET
+  /events`](/api/get-events), [`GET /streams`](/api/get-streams),
+  [`GET /streams/{stream_id}`](/api/get-stream-by-id): Added a new
+  field `is_recently_active` to stream objects as a new deterministic
+  source of truth for `demote_inactive_streams` activity decisions.
+
 **Feature level 322**
 
 * [`POST /invites`](/api/send-invites), [`POST

puppet/zulip/manifests/app_frontend_once.pp

@@ -36,6 +36,11 @@ class zulip::app_frontend_once {
   }
 
   # Daily
+  zulip::cron { 'update-channel-recently-active-status':
+    hour   => '1',
+    minute => '40',
+    manage => 'update_channel_recently_active_status',
+  }
   zulip::cron { 'soft-deactivate-users':
     hour   => '5',
     minute => '0',

version.py

@@ -34,7 +34,7 @@ DESKTOP_WARNING_VERSION = "5.9.3"
 # new level means in api_docs/changelog.md, as well as "**Changes**"
 # entries in the endpoint's documentation in `zulip.yaml`.
 
-API_FEATURE_LEVEL = 322  # Last bumped for adding users to groups using invitations
+API_FEATURE_LEVEL = 323  # Last bumped for "GET /streams `is_recently_active`"
 
 # Bump the minor PROVISION_VERSION to indicate that folks should provision
 # only when going from an old version of the code to a newer version. Bump

zerver/actions/message_send.py

@@ -1158,6 +1158,7 @@ def do_send_messages(
 
             # assert needed because stubs for django are missing
             assert send_request.stream is not None
+            stream_update_fields = []
             if send_request.stream.is_public():
                 event["realm_id"] = send_request.stream.realm_id
                 event["stream_name"] = send_request.stream.name
@@ -1165,7 +1166,12 @@ def do_send_messages(
                 event["invite_only"] = True
             if send_request.stream.first_message_id is None:
                 send_request.stream.first_message_id = send_request.message.id
-                send_request.stream.save(update_fields=["first_message_id"])
+                stream_update_fields.append("first_message_id")
+            if not send_request.stream.is_recently_active:
+                send_request.stream.is_recently_active = True
+                stream_update_fields.append("is_recently_active")
+            if len(stream_update_fields) > 0:
+                send_request.stream.save(update_fields=stream_update_fields)
 
             # Performance note: This check can theoretically do
             # database queries in a loop if many messages are being

zerver/actions/streams.py

@@ -396,6 +396,7 @@ def send_subscription_add_events(
                 date_created=stream_dict["date_created"],
                 description=stream_dict["description"],
                 first_message_id=stream_dict["first_message_id"],
+                is_recently_active=stream_dict["is_recently_active"],
                 history_public_to_subscribers=stream_dict["history_public_to_subscribers"],
                 invite_only=stream_dict["invite_only"],
                 is_web_public=stream_dict["is_web_public"],

zerver/lib/event_schema.py

@@ -69,6 +69,7 @@ default_stream_fields = [
     ("date_created", int),
     ("description", str),
     ("first_message_id", OptionalType(int)),
+    ("is_recently_active", bool),
     ("history_public_to_subscribers", bool),
     ("invite_only", bool),
     ("is_announcement_only", bool),
@@ -1454,6 +1455,9 @@ def check_stream_update(
     elif prop == "first_message_id":
         assert extra_keys == set()
         assert isinstance(value, int)
+    elif prop == "is_recently_active":
+        assert extra_keys == set()
+        assert isinstance(value, bool)
     else:
         raise AssertionError(f"Unknown property: {prop}")
 

zerver/lib/import_realm.py

@@ -3,7 +3,7 @@ import logging
 import os
 import shutil
 from concurrent.futures import ProcessPoolExecutor, as_completed
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from difflib import unified_diff
 from typing import Any
 
@@ -46,7 +46,7 @@ from zerver.lib.partial import partial
 from zerver.lib.push_notifications import sends_notifications_directly
 from zerver.lib.remote_server import maybe_enqueue_audit_log_upload
 from zerver.lib.server_initialization import create_internal_realm, server_initialized
-from zerver.lib.streams import render_stream_description
+from zerver.lib.streams import render_stream_description, update_stream_active_status_for_realm
 from zerver.lib.thumbnail import THUMBNAIL_ACCEPT_IMAGE_TYPES, BadImageError, maybe_thumbnail
 from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.lib.upload import ensure_avatar_image, sanitize_name, upload_backend, upload_emoji_image
@@ -1753,6 +1753,12 @@ def do_import_realm(import_dir: Path, subdomain: str, processes: int = 1) -> Rea
     realm.deactivated = data["zerver_realm"][0]["deactivated"]
     realm.save()
 
+    # If realm is active, update the stream active status.
+    if not realm.deactivated:
+        number_of_days = Stream.LAST_ACTIVITY_DAYS_BEFORE_FOR_ACTIVE
+        date_days_ago = timezone_now() - timedelta(days=number_of_days)
+        update_stream_active_status_for_realm(realm, date_days_ago)
+
     # This helps to have an accurate user count data for the billing
     # system if someone tries to signup just after doing import.
     RealmAuditLog.objects.create(

zerver/lib/streams.py

@@ -1,4 +1,5 @@
 from collections.abc import Collection
+from datetime import datetime, timedelta
 from typing import TypedDict
 
 from django.db import transaction
@@ -29,6 +30,7 @@ from zerver.lib.user_groups import (
 from zerver.models import (
     DefaultStreamGroup,
     GroupGroupMembership,
+    Message,
     NamedUserGroup,
     Realm,
     RealmAuditLog,
@@ -931,6 +933,7 @@ def stream_to_dict(
         date_created=datetime_to_timestamp(stream.date_created),
         description=stream.description,
         first_message_id=stream.first_message_id,
+        is_recently_active=stream.is_recently_active,
         history_public_to_subscribers=stream.history_public_to_subscribers,
         invite_only=stream.invite_only,
         is_web_public=stream.is_web_public,
@@ -1132,3 +1135,42 @@ def get_subscribed_private_streams_for_user(user_profile: UserProfile) -> QueryS
         .filter(subscribed=True)
     )
     return subscribed_private_streams
+
+
+@transaction.atomic(durable=True)
+def update_stream_active_status_for_realm(realm: Realm, date_days_ago: datetime) -> int:
+    active_stream_ids = (
+        Message.objects.filter(
+            date_sent__gte=date_days_ago, recipient__type=Recipient.STREAM, realm=realm
+        )
+        .values_list("recipient__type_id", flat=True)
+        .distinct()
+    )
+    streams_to_mark_inactive = Stream.objects.filter(is_recently_active=True, realm=realm).exclude(
+        id__in=active_stream_ids
+    )
+
+    # Send events to notify the users about the change in the stream's active status.
+    for stream in streams_to_mark_inactive:
+        event = dict(
+            type="stream",
+            op="update",
+            property="is_recently_active",
+            value=False,
+            stream_id=stream.id,
+            name=stream.name,
+        )
+        send_event_on_commit(stream.realm, event, active_user_ids(stream.realm_id))
+
+    count = streams_to_mark_inactive.update(is_recently_active=False)
+    return count
+
+
+def check_update_all_streams_active_status(
+    days: int = Stream.LAST_ACTIVITY_DAYS_BEFORE_FOR_ACTIVE,
+) -> int:
+    date_days_ago = timezone_now() - timedelta(days=days)
+    count = 0
+    for realm in Realm.objects.filter(deactivated=False):
+        count += update_stream_active_status_for_realm(realm, date_days_ago)
+    return count

zerver/lib/subscription_info.py

@@ -57,6 +57,7 @@ def get_web_public_subs(realm: Realm) -> SubscriptionInfo:
         date_created = datetime_to_timestamp(stream.date_created)
         description = stream.description
         first_message_id = stream.first_message_id
+        is_recently_active = stream.is_recently_active
         history_public_to_subscribers = stream.history_public_to_subscribers
         invite_only = stream.invite_only
         is_announcement_only = stream.stream_post_policy == Stream.STREAM_POST_POLICY_ADMINS
@@ -93,6 +94,7 @@ def get_web_public_subs(realm: Realm) -> SubscriptionInfo:
             desktop_notifications=desktop_notifications,
             email_notifications=email_notifications,
             first_message_id=first_message_id,
+            is_recently_active=is_recently_active,
             history_public_to_subscribers=history_public_to_subscribers,
             in_home_view=in_home_view,
             invite_only=invite_only,
@@ -165,6 +167,7 @@ def build_stream_api_dict(
         stream_post_policy=raw_stream_dict["stream_post_policy"],
         stream_weekly_traffic=stream_weekly_traffic,
         is_announcement_only=is_announcement_only,
+        is_recently_active=raw_stream_dict["is_recently_active"],
     )
 
 
@@ -190,6 +193,7 @@ def build_stream_dict_for_sub(
     stream_post_policy = stream_dict["stream_post_policy"]
     stream_weekly_traffic = stream_dict["stream_weekly_traffic"]
     is_announcement_only = stream_dict["is_announcement_only"]
+    is_recently_active = stream_dict["is_recently_active"]
 
     # Handle Subscription.API_FIELDS.
     color = sub_dict["color"]
@@ -217,6 +221,7 @@ def build_stream_dict_for_sub(
         desktop_notifications=desktop_notifications,
         email_notifications=email_notifications,
         first_message_id=first_message_id,
+        is_recently_active=is_recently_active,
         history_public_to_subscribers=history_public_to_subscribers,
         in_home_view=in_home_view,
         invite_only=invite_only,
@@ -245,6 +250,7 @@ def build_stream_dict_for_never_sub(
     date_created = datetime_to_timestamp(raw_stream_dict["date_created"])
     description = raw_stream_dict["description"]
     first_message_id = raw_stream_dict["first_message_id"]
+    is_recently_active = raw_stream_dict["is_recently_active"]
     history_public_to_subscribers = raw_stream_dict["history_public_to_subscribers"]
     invite_only = raw_stream_dict["invite_only"]
     is_web_public = raw_stream_dict["is_web_public"]
@@ -276,6 +282,7 @@ def build_stream_dict_for_never_sub(
         date_created=date_created,
         description=description,
         first_message_id=first_message_id,
+        is_recently_active=is_recently_active,
         history_public_to_subscribers=history_public_to_subscribers,
         invite_only=invite_only,
         is_announcement_only=is_announcement_only,

zerver/lib/types.py

@@ -156,6 +156,7 @@ class RawStreamDict(TypedDict):
     deactivated: bool
     description: str
     first_message_id: int | None
+    is_recently_active: bool
     history_public_to_subscribers: bool
     id: int
     invite_only: bool
@@ -199,6 +200,7 @@ class SubscriptionStreamDict(TypedDict):
     desktop_notifications: bool | None
     email_notifications: bool | None
     first_message_id: int | None
+    is_recently_active: bool
     history_public_to_subscribers: bool
     in_home_view: bool
     invite_only: bool
@@ -225,6 +227,7 @@ class NeverSubscribedStreamDict(TypedDict):
     date_created: int
     description: str
     first_message_id: int | None
+    is_recently_active: bool
     history_public_to_subscribers: bool
     invite_only: bool
     is_announcement_only: bool
@@ -250,6 +253,7 @@ class DefaultStreamDict(TypedDict):
     date_created: int
     description: str
     first_message_id: int | None
+    is_recently_active: bool
     history_public_to_subscribers: bool
     invite_only: bool
     is_web_public: bool

zerver/management/commands/update_channel_recently_active_status.py

@@ -0,0 +1,22 @@
+import logging
+from typing import Any
+
+from django.conf import settings
+from typing_extensions import override
+
+from zerver.lib.logging_util import log_to_file
+from zerver.lib.management import ZulipBaseCommand
+from zerver.lib.streams import check_update_all_streams_active_status
+
+## Logging setup ##
+logger = logging.getLogger(__name__)
+log_to_file(logger, settings.DIGEST_LOG_PATH)
+
+
+class Command(ZulipBaseCommand):
+    help = """Update the `Stream.is_recently_active` field to False for channels whose message history has aged to the point where it is no longer recently active."""
+
+    @override
+    def handle(self, *args: Any, **options: Any) -> None:
+        count = check_update_all_streams_active_status()
+        logger.info("Marked %s channels as not recently active.", count)

zerver/migrations/0631_stream_is_recently_active.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.9 on 2024-11-27 04:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("zerver", "0630_multiuseinvite_groups_preregistrationuser_groups"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="stream",
+            name="is_recently_active",
+            field=models.BooleanField(db_default=True, default=True),
+        ),
+    ]

zerver/models/streams.py

@@ -132,6 +132,11 @@ class Stream(models.Model):
     # stream based on what messages they have cached.
     first_message_id = models.IntegerField(null=True, db_index=True)
 
+    LAST_ACTIVITY_DAYS_BEFORE_FOR_ACTIVE = 180
+
+    # Whether a message has been sent to this stream in the last X days.
+    is_recently_active = models.BooleanField(default=True, db_default=True)
+
     stream_permission_group_settings = {
         "can_remove_subscribers_group": GroupPermissionSetting(
             require_system_group=False,
@@ -184,6 +189,7 @@ class Stream(models.Model):
         "rendered_description",
         "stream_post_policy",
         "can_remove_subscribers_group_id",
+        "is_recently_active",
     ]
 
     def to_dict(self) -> DefaultStreamDict:
@@ -203,6 +209,7 @@ class Stream(models.Model):
             stream_id=self.id,
             stream_post_policy=self.stream_post_policy,
             is_announcement_only=self.stream_post_policy == Stream.STREAM_POST_POLICY_ADMINS,
+            is_recently_active=self.is_recently_active,
         )
 
 

zerver/openapi/zulip.yaml

@@ -703,6 +703,7 @@ paths:
                                         "stream_post_policy": 1,
                                         "history_public_to_subscribers": true,
                                         "first_message_id": null,
+                                        "is_recently_active": true,
                                         "message_retention_days": null,
                                         "is_announcement_only": false,
                                         "color": "#76ce90",
@@ -1348,6 +1349,7 @@ paths:
                                         "stream_post_policy": 1,
                                         "history_public_to_subscribers": false,
                                         "first_message_id": null,
+                                        "is_recently_active": true,
                                         "message_retention_days": null,
                                         "is_announcement_only": false,
                                         "can_remove_subscribers_group": 2,
@@ -1407,6 +1409,7 @@ paths:
                                         "stream_post_policy": 1,
                                         "history_public_to_subscribers": false,
                                         "first_message_id": null,
+                                        "is_recently_active": true,
                                         "message_retention_days": null,
                                         "is_announcement_only": false,
                                         "can_remove_subscribers_group": 2,
@@ -2033,6 +2036,7 @@ paths:
                                               "stream_post_policy": 1,
                                               "history_public_to_subscribers": true,
                                               "first_message_id": 1,
+                                              "is_recently_active": true,
                                               "message_retention_days": null,
                                               "is_announcement_only": false,
                                               "can_remove_subscribers_group": 2,
@@ -2048,6 +2052,7 @@ paths:
                                               "invite_only": false,
                                               "is_web_public": false,
                                               "stream_post_policy": 1,
+                                              "is_recently_active": true,
                                               "history_public_to_subscribers": true,
                                               "first_message_id": 4,
                                               "message_retention_days": null,
@@ -2067,6 +2072,7 @@ paths:
                                               "stream_post_policy": 1,
                                               "history_public_to_subscribers": true,
                                               "first_message_id": 6,
+                                              "is_recently_active": true,
                                               "message_retention_days": null,
                                               "is_announcement_only": false,
                                               "can_remove_subscribers_group": 2,
@@ -2115,6 +2121,7 @@ paths:
                                         "stream_post_policy": 1,
                                         "history_public_to_subscribers": true,
                                         "first_message_id": 1,
+                                        "is_recently_active": true,
                                         "message_retention_days": null,
                                         "is_announcement_only": false,
                                         "can_remove_subscribers_group": 2,
@@ -14881,6 +14888,7 @@ paths:
                                 history_public_to_subscribers: {}
                                 first_message_id:
                                   nullable: true
+                                is_recently_active: {}
                                 is_announcement_only: {}
                                 can_remove_subscribers_group: {}
                                 stream_weekly_traffic:
@@ -19710,6 +19718,7 @@ paths:
                                 history_public_to_subscribers: {}
                                 first_message_id:
                                   nullable: true
+                                is_recently_active: {}
                                 is_announcement_only: {}
                                 can_remove_subscribers_group: {}
                                 stream_weekly_traffic:
@@ -19754,6 +19763,7 @@ paths:
                                 - is_announcement_only
                                 - can_remove_subscribers_group
                                 - stream_weekly_traffic
+                                - is_recently_active
                     example:
                       {
                         "msg": "",
@@ -19766,6 +19776,7 @@ paths:
                               "date_created": 1691057093,
                               "description": "A private channel",
                               "first_message_id": 18,
+                              "is_recently_active": true,
                               "history_public_to_subscribers": false,
                               "invite_only": true,
                               "is_announcement_only": false,
@@ -19785,6 +19796,7 @@ paths:
                               "date_created": 1691057093,
                               "description": "A default public channel",
                               "first_message_id": 21,
+                              "is_recently_active": true,
                               "history_public_to_subscribers": true,
                               "invite_only": false,
                               "is_announcement_only": false,
@@ -19851,6 +19863,7 @@ paths:
                           {
                             "description": "A Scandinavian country",
                             "first_message_id": 1,
+                            "is_recently_active": true,
                             "history_public_to_subscribers": true,
                             "date_created": 1691057093,
                             "creator_id": null,
@@ -21719,6 +21732,7 @@ components:
             history_public_to_subscribers: {}
             first_message_id:
               nullable: true
+            is_recently_active: {}
             is_announcement_only: {}
             can_remove_subscribers_group: {}
             stream_weekly_traffic:
@@ -21750,6 +21764,7 @@ components:
             - message_retention_days
             - history_public_to_subscribers
             - first_message_id
+            - is_recently_active
             - is_announcement_only
             - can_remove_subscribers_group
             - stream_weekly_traffic
@@ -21774,6 +21789,7 @@ components:
             history_public_to_subscribers: {}
             first_message_id:
               nullable: true
+            is_recently_active: {}
             is_announcement_only: {}
             can_remove_subscribers_group: {}
           required:
@@ -21790,6 +21806,7 @@ components:
             - message_retention_days
             - history_public_to_subscribers
             - first_message_id
+            - is_recently_active
             - is_announcement_only
             - can_remove_subscribers_group
     BasicChannelBase:
@@ -21909,6 +21926,16 @@ components:
             Is `null` for channels with no message history.
 
             **Changes**: New in Zulip 2.1.0.
+        is_recently_active:
+          type: boolean
+          description: |
+            Whether the channel has recent message activity. Clients should use this to implement
+            [sorting inactive channels to the bottom](/help/manage-inactive-channels)
+            if `demote_inactive_streams` is enabled.
+
+            **Changes**: New in Zulip 10.0 (feature level 323). Previously, clients implemented the
+            demote_inactive_streams from local message history, resulting in a choppy loading
+            experience.
         is_announcement_only:
           type: boolean
           deprecated: true
@@ -22915,6 +22942,15 @@ components:
             has older history that can be accessed.
 
             Is `null` for channels with no message history.
+        is_recently_active:
+          type: boolean
+          description: |
+            Whether the channel has recent message activity. Clients should use this to implement
+            [sorting inactive channels to the bottom](/help/manage-inactive-channels).
+
+            **Changes**: New in Zulip 10.0 (feature level 323). Previously, clients implemented the
+            demote_inactive_streams from local message history, resulting in a choppy loading
+            experience.
         stream_weekly_traffic:
           type: integer
           nullable: true

zerver/tests/test_events.py

@@ -210,6 +210,7 @@ from zerver.lib.events import apply_events, fetch_initial_state_data, post_proce
 from zerver.lib.markdown import render_message_markdown
 from zerver.lib.mention import MentionBackend, MentionData
 from zerver.lib.muted_users import get_mute_object
+from zerver.lib.streams import check_update_all_streams_active_status
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import (
     create_dummy_file,
@@ -3478,6 +3479,19 @@ class NormalActionsTest(BaseAction):
             is_legacy=False,
         )
 
+    def test_check_update_all_streams_active_status(self) -> None:
+        hamlet = self.example_user("hamlet")
+        self.subscribe(hamlet, "test_stream1")
+        stream = get_stream("test_stream1", self.user_profile.realm)
+
+        # Delete all messages in the stream so that it becomes inactive.
+        Message.objects.filter(recipient__type_id=stream.id, realm=stream.realm).delete()
+
+        with self.verify_action() as events:
+            check_update_all_streams_active_status()
+
+        check_stream_update("events[0]", events[0])
+
     def test_do_delete_message_personal(self) -> None:
         msg_id = self.send_personal_message(
             self.example_user("cordelia"),

zerver/tests/test_message_send.py

@@ -2392,6 +2392,21 @@ class StreamMessagesTest(ZulipTestCase):
         )
         self.assertEqual(recent_conversation["max_message_id"], message2_id)
 
+    def test_stream_becomes_active_on_message_send(self) -> None:
+        # Mark a stream as inactive
+        stream = self.make_stream("inactive_stream")
+        stream.is_recently_active = False
+        stream.save()
+        self.assertEqual(stream.is_recently_active, False)
+
+        # Send a message to the stream
+        sender = self.example_user("hamlet")
+        self.subscribe(sender, stream.name)
+        self.send_stream_message(sender, stream.name)
+        # The stream should now be active
+        stream.refresh_from_db()
+        self.assertEqual(stream.is_recently_active, True)
+
 
 class PersonalMessageSendTest(ZulipTestCase):
     def test_personal_to_self(self) -> None: