retention: Archive cross-realm personal messages.

We can simply archive cross-realm personal messages according to the
retention policy of the recipient's realm. It requires adding another
message-archiving query for this case however.

What remains is to figure out how to treat cross-realm huddle messages.

zerver/lib/retention.py

@@ -166,11 +166,38 @@ def move_expired_personal_and_huddle_messages_to_archive(realm: Realm,
     assert realm.message_retention_days is not None
     check_date = timezone_now() - timedelta(days=realm.message_retention_days)
 
-    return run_archiving_in_chunks(query, type=ArchiveTransaction.RETENTION_POLICY_BASED, realm=realm,
+    message_count = run_archiving_in_chunks(query, type=ArchiveTransaction.RETENTION_POLICY_BASED,
+                                            realm=realm,
                                             cross_realm_bot_ids=cross_realm_bot_ids,
                                             realm_id=realm.id, recipient_types=recipient_types,
                                             check_date=check_date.isoformat(), chunk_size=chunk_size)
 
+    # Archive cross-realm personal messages to users in the realm:
+    # Note: Cross-realm huddle message aren't handled yet, they remain an issue
+    # that should be addressed.
+    query = """
+    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_transaction_id)
+        SELECT {src_fields}, {archive_transaction_id}
+        FROM zerver_message
+        INNER JOIN zerver_recipient ON zerver_recipient.id = zerver_message.recipient_id
+        INNER JOIN zerver_userprofile recipient_profile ON recipient_profile.id = zerver_recipient.type_id
+        INNER JOIN zerver_userprofile sender_profile ON sender_profile.id = zerver_message.sender_id
+        WHERE sender_profile.id IN {cross_realm_bot_ids}
+            AND recipient_profile.realm_id = {realm_id}
+            AND zerver_recipient.type = {recipient_personal}
+            AND zerver_message.pub_date < '{check_date}'
+        LIMIT {chunk_size}
+    ON CONFLICT (id) DO UPDATE SET archive_transaction_id = {archive_transaction_id}
+    RETURNING id
+    """
+    message_count += run_archiving_in_chunks(query, type=ArchiveTransaction.RETENTION_POLICY_BASED,
+                                             realm=realm,
+                                             cross_realm_bot_ids=cross_realm_bot_ids,
+                                             realm_id=realm.id, recipient_personal=Recipient.PERSONAL,
+                                             check_date=check_date.isoformat(), chunk_size=chunk_size)
+
+    return message_count
+
 def move_models_with_message_key_to_archive(msg_ids: List[int]) -> None:
     assert len(msg_ids) > 0
 

zerver/tests/test_retention.py

@@ -9,7 +9,7 @@ from django.utils.timezone import now as timezone_now
 from zerver.lib.actions import internal_send_private_message, do_add_submessage
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.upload import create_attachment
-from zerver.models import (Message, Realm, UserProfile, Stream, ArchivedUserMessage, SubMessage,
+from zerver.models import (Message, Realm, Stream, ArchivedUserMessage, SubMessage,
                            ArchivedMessage, Attachment, ArchivedAttachment, UserMessage,
                            Reaction, ArchivedReaction, ArchivedSubMessage, ArchiveTransaction,
                            get_realm, get_user_profile_by_email, get_stream, get_system_bot)
@@ -103,15 +103,15 @@ class ArchiveMessagesTestingBase(RetentionTestingBase):
         self._change_messages_pub_date(msg_ids, pub_date)
         return msg_ids
 
-    def _send_cross_realm_message(self) -> int:
+    def _send_cross_realm_personal_message(self) -> int:
         # Send message from bot to users from different realm.
         bot_email = 'notification-bot@zulip.com'
         get_user_profile_by_email(bot_email)
-        mit_user = UserProfile.objects.filter(realm=self.mit_realm).first()
+        zulip_user = self.example_user("hamlet")
         msg_id = internal_send_private_message(
-            realm=mit_user.realm,
+            realm=self.zulip_realm,
             sender=get_system_bot(bot_email),
-            recipient_user=mit_user,
+            recipient_user=zulip_user,
             content='test message',
         )
         assert msg_id is not None
@@ -246,19 +246,15 @@ class TestArchiveMessagesGeneral(ArchiveMessagesTestingBase):
         archive_messages()
         self._verify_archive_data([msg_id], usermsg_ids)
 
-    def test_cross_realm_messages_not_archived(self) -> None:
+    def test_cross_realm_personal_message_archiving(self) -> None:
-        """Check that cross-realm messages don't get archived or deleted."""
+        """Check that cross-realm personal messages get correctly archived. """
-        msg_id = self._send_cross_realm_message()
+        msg_ids = [self._send_cross_realm_personal_message() for i in range(1, 7)]
-        # Make the message expired on both realms:
+        usermsg_ids = self._get_usermessage_ids(msg_ids)
-        self._change_messages_pub_date([msg_id], timezone_now() - timedelta(MIT_REALM_DAYS+1))
+        # Make the message expired on the recipient's realm:
+        self._change_messages_pub_date(msg_ids, timezone_now() - timedelta(ZULIP_REALM_DAYS+1))
 
         archive_messages()
-
+        self._verify_archive_data(msg_ids, usermsg_ids)
-        self.assertEqual(ArchivedMessage.objects.count(), 0)
-        self.assertEqual(ArchivedUserMessage.objects.count(), 0)
-
-        self.assertEqual(UserMessage.objects.filter(message_id=msg_id).count(), 2)
-        self.assertTrue(Message.objects.filter(id=msg_id).exists())
 
     def test_archiving_interrupted(self) -> None:
         """ Check that queries get rolled back to a consistent state
@@ -297,14 +293,14 @@ class TestArchiveMessagesGeneral(ArchiveMessagesTestingBase):
         # Change some Zulip messages to be expired:
         expired_zulip_msg_ids = self._make_expired_zulip_messages(7)
 
-        expired_crossrealm_msg_id = self._send_cross_realm_message()
+        expired_crossrealm_msg_id = self._send_cross_realm_personal_message()
-        # Make the message expired on both realms:
+        # Make the message expired in the recipient's realm:
         self._change_messages_pub_date(
             [expired_crossrealm_msg_id],
-            timezone_now() - timedelta(MIT_REALM_DAYS+1)
+            timezone_now() - timedelta(ZULIP_REALM_DAYS+1)
         )
 
-        expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids
+        expired_msg_ids = expired_mit_msg_ids + expired_zulip_msg_ids + [expired_crossrealm_msg_id]
         expired_usermsg_ids = self._get_usermessage_ids(expired_msg_ids)
 
         archive_messages(chunk_size=2)  # Specify low chunk_size to test batching.