thumbnail: Resolve a race condition when rendering messages.

Messages are rendered outside of a transaction, for performance
reasons, and then sent inside of one.  This opens thumbnailing up to a
race where the thumbnails have not yet been written when the message
is rendered, but the message has not been sent when thumbnailing
completes, causing `rewrite_thumbnailed_images` to be a no-op and the
message being left with a spinner which never resolves.

Explicitly lock and use he ImageAttachment data inside the
message-sending transaction, to rewrite the message content with the
latest information about the existing thumbnails.

Despite the thumbnailing worker taking a lock on Message rows to
update them, this does not lead to deadlocks -- the INSERT of the
Message rows happens in a transaction, ensuring that either the
message rending blocks the thumbnailing until the Message row is
created, or that the `rewrite_thumbnailed_images` and Message INSERT
waits until thumbnailing is complete (and updated no Message rows).

zerver/tests/test_markdown_thumbnail.py

@@ -4,13 +4,22 @@ from unittest.mock import patch
 import pyvips
 
 from zerver.actions.message_delete import do_delete_messages
+from zerver.actions.message_send import check_message, do_send_messages
+from zerver.lib.addressee import Addressee
 from zerver.lib.camo import get_camo_url
 from zerver.lib.markdown import render_message_markdown
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import get_test_image_file, read_test_image_file
 from zerver.lib.thumbnail import ThumbnailFormat
 from zerver.lib.upload import upload_message_attachment
-from zerver.models import ArchivedAttachment, ArchivedMessage, Attachment, ImageAttachment, Message
+from zerver.models import (
+    ArchivedAttachment,
+    ArchivedMessage,
+    Attachment,
+    Client,
+    ImageAttachment,
+    Message,
+)
 from zerver.models.clients import get_client
 from zerver.worker.thumbnail import ensure_thumbnails
 
@@ -334,3 +343,42 @@ class MarkdownThumbnailTest(ZulipTestCase):
             private_message_id,
             f'<p>This <a href="/user_uploads/{path_id}">image</a> is private</p>\n{rendered_thumb}',
         )
+
+    def test_thumbnail_race(self) -> None:
+        """Test what happens when thumbnailing happens between rendering and sending"""
+        path_id = self.upload_image("img.png")
+
+        self.assert_length(ImageAttachment.objects.get(path_id=path_id).thumbnail_metadata, 0)
+
+        # Render, but do not send, the message referencing the image.
+        # This will render as a spinner, since the thumbnail has not
+        # been generated yet.
+        send_request = check_message(
+            self.example_user("othello"),
+            Client.objects.get_or_create(name="test suite")[0],
+            Addressee.for_stream_name("Verona", "test"),
+            f"[image](/user_uploads/{path_id})",
+        )
+        expected = (
+            f'<p><a href="/user_uploads/{path_id}">image</a></p>\n'
+            f'<div class="message_inline_image"><a href="/user_uploads/{path_id}" title="image">'
+            '<img class="image-loading-placeholder" src="/static/images/loading/loader-black.svg"></a></div>'
+        )
+        self.assertEqual(send_request.message.rendered_content, expected)
+
+        # Thumbnail the image.  The message does not exist yet, so
+        # nothing is re-written.
+        ensure_thumbnails(ImageAttachment.objects.get(path_id=path_id))
+
+        # Send the message; this should re-check the ImageAttachment
+        # data, find the thumbnails, and update the rendered_content
+        # to no longer contain a spinner.
+        message_id = do_send_messages([send_request])[0].message_id
+
+        rendered_thumb = (
+            f'<div class="message_inline_image"><a href="/user_uploads/{path_id}" title="image">'
+            f'<img data-original-dimensions="128x128" src="/user_uploads/thumbnail/{path_id}/840x560.webp"></a></div>'
+        )
+        self.assert_message_content_is(
+            message_id, f'<p><a href="/user_uploads/{path_id}">image</a></p>\n{rendered_thumb}'
+        )