diff --git a/zerver/lib/actions.py b/zerver/lib/actions.py index 612dde71b6..386a6dbd88 100644 --- a/zerver/lib/actions.py +++ b/zerver/lib/actions.py @@ -96,7 +96,7 @@ from zerver.lib.export import get_realm_exports_serialized from zerver.lib.external_accounts import DEFAULT_EXTERNAL_ACCOUNTS from zerver.lib.hotspots import get_next_hotspots from zerver.lib.i18n import get_language_name -from zerver.lib.markdown import topic_links +from zerver.lib.markdown import MessageRenderingResult, topic_links from zerver.lib.markdown import version as markdown_version from zerver.lib.mention import MentionData from zerver.lib.message import ( @@ -1413,10 +1413,10 @@ def render_incoming_message( realm: Realm, mention_data: Optional[MentionData] = None, email_gateway: bool = False, -) -> str: +) -> MessageRenderingResult: realm_alert_words_automaton = get_alert_word_automaton(realm) try: - rendered_content = render_markdown( + rendering_result = render_markdown( message=message, content=content, realm=realm, @@ -1426,7 +1426,7 @@ def render_incoming_message( ) except MarkdownRenderingException: raise JsonableError(_("Unable to render message")) - return rendered_content + return rendering_result class RecipientInfoResult(TypedDict): @@ -1780,7 +1780,7 @@ def build_message_send_dict( # Render our message_dicts. assert message.rendered_content is None - rendered_content = render_incoming_message( + rendering_result = render_incoming_message( message, message.content, info["active_user_ids"], @@ -1788,20 +1788,20 @@ def build_message_send_dict( mention_data=mention_data, email_gateway=email_gateway, ) - message.rendered_content = rendered_content + message.rendered_content = rendering_result.rendered_content message.rendered_content_version = markdown_version - links_for_embed = message.links_for_preview + links_for_embed = rendering_result.links_for_preview # Add members of the mentioned user groups into `mentions_user_ids`. - for group_id in message.mentions_user_group_ids: + for group_id in rendering_result.mentions_user_group_ids: members = mention_data.get_group_members(group_id) - message.mentions_user_ids.update(members) + rendering_result.mentions_user_ids.update(members) # Only send data to Tornado about wildcard mentions if message # rendering determined the message had an actual wildcard # mention in it (and not e.g. wildcard mention syntax inside a # code block). - if message.mentions_wildcard: + if rendering_result.mentions_wildcard: wildcard_mention_user_ids = info["wildcard_mention_user_ids"] else: wildcard_mention_user_ids = set() @@ -1812,7 +1812,7 @@ def build_message_send_dict( who were directly mentioned in this message as eligible to get UserMessage rows. """ - mentioned_user_ids = message.mentions_user_ids + mentioned_user_ids = rendering_result.mentions_user_ids default_bot_user_ids = info["default_bot_user_ids"] mentioned_bot_user_ids = default_bot_user_ids & mentioned_user_ids info["um_eligible_user_ids"] |= mentioned_bot_user_ids @@ -1824,6 +1824,7 @@ def build_message_send_dict( realm=realm, mention_data=mention_data, message=message, + rendering_result=rendering_result, active_user_ids=info["active_user_ids"], online_push_user_ids=info["online_push_user_ids"], stream_push_user_ids=info["stream_push_user_ids"], @@ -1866,7 +1867,7 @@ def do_send_messages( # Claim attachments in message for send_request in send_message_requests: if do_claim_attachments( - send_request.message, send_request.message.potential_attachment_path_ids + send_request.message, send_request.rendering_result.potential_attachment_path_ids ): send_request.message.has_attachment = True send_request.message.save(update_fields=["has_attachment"]) @@ -1875,7 +1876,7 @@ def do_send_messages( for send_request in send_message_requests: # Service bots (outgoing webhook bots and embedded bots) don't store UserMessage rows; # they will be processed later. - mentioned_user_ids = send_request.message.mentions_user_ids + mentioned_user_ids = send_request.rendering_result.mentions_user_ids # Extend the set with users who have muted the sender. mark_as_read_for_users = send_request.muted_sender_user_ids @@ -1883,6 +1884,7 @@ def do_send_messages( user_messages = create_user_messages( message=send_request.message, + rendering_result=send_request.rendering_result, um_eligible_user_ids=send_request.um_eligible_user_ids, long_term_idle_user_ids=send_request.long_term_idle_user_ids, stream_push_user_ids=send_request.stream_push_user_ids, @@ -2068,6 +2070,7 @@ class UserMessageLite: def create_user_messages( message: Message, + rendering_result: MessageRenderingResult, um_eligible_user_ids: AbstractSet[int], long_term_idle_user_ids: AbstractSet[int], stream_push_user_ids: AbstractSet[int], @@ -2077,12 +2080,12 @@ def create_user_messages( ) -> List[UserMessageLite]: # These properties on the Message are set via # render_markdown by code in the Markdown inline patterns - ids_with_alert_words = message.user_ids_with_alert_words + ids_with_alert_words = rendering_result.user_ids_with_alert_words sender_id = message.sender.id is_stream_message = message.is_stream_message() base_flags = 0 - if message.mentions_wildcard: + if rendering_result.mentions_wildcard: base_flags |= UserMessage.flags.wildcard_mentioned if message.recipient.type in [Recipient.HUDDLE, Recipient.PERSONAL]: base_flags |= UserMessage.flags.is_private @@ -2891,7 +2894,7 @@ def check_update_message( if (timezone_now() - message.date_sent) > datetime.timedelta(seconds=deadline_seconds): raise JsonableError(_("The time limit for editing this message's topic has passed")) - rendered_content = None + rendering_result = None links_for_embed: Set[str] = set() prior_mention_user_ids: Set[int] = set() mention_data: Optional[MentionData] = None @@ -2911,14 +2914,14 @@ def check_update_message( # the cross-realm bots never edit messages, this should be # always correct. # Note: If rendering fails, the called code will raise a JsonableError. - rendered_content = render_incoming_message( + rendering_result = render_incoming_message( message, content, user_info["message_user_ids"], user_profile.realm, mention_data=mention_data, ) - links_for_embed |= message.links_for_preview + links_for_embed |= rendering_result.links_for_preview new_stream = None number_changed = 0 @@ -2948,7 +2951,7 @@ def check_update_message( send_notification_to_old_thread, send_notification_to_new_thread, content, - rendered_content, + rendering_result, prior_mention_user_ids, mention_data, ) @@ -3254,7 +3257,7 @@ def check_message( email_gateway=email_gateway, ) - if stream is not None and message_send_dict.message.mentions_wildcard: + if stream is not None and message_send_dict.rendering_result.mentions_wildcard: if not wildcard_mention_allowed(sender, stream): raise JsonableError( _("You do not have permission to use wildcard mentions in this stream.") @@ -5758,10 +5761,12 @@ def get_user_info_for_message_updates(message_id: int) -> MessageUpdateUserInfoR ) -def update_user_message_flags(message: Message, ums: Iterable[UserMessage]) -> None: - wildcard = message.mentions_wildcard - mentioned_ids = message.mentions_user_ids - ids_with_alert_words = message.user_ids_with_alert_words +def update_user_message_flags( + rendering_result: MessageRenderingResult, ums: Iterable[UserMessage] +) -> None: + wildcard = rendering_result.mentions_wildcard + mentioned_ids = rendering_result.mentions_user_ids + ids_with_alert_words = rendering_result.user_ids_with_alert_words changed_ums: Set[UserMessage] = set() def update_flag(um: UserMessage, should_set: bool, flag: int) -> None: @@ -5810,16 +5815,17 @@ def do_update_embedded_data( user_profile: UserProfile, message: Message, content: Optional[str], - rendered_content: Optional[str], + rendering_result: MessageRenderingResult, ) -> None: event: Dict[str, Any] = {"type": "update_message", "message_id": message.id} changed_messages = [message] + rendered_content: Optional[str] = None ums = UserMessage.objects.filter(message=message.id) if content is not None: - update_user_message_flags(message, ums) - message.content = content + update_user_message_flags(rendering_result, ums) + rendered_content = rendering_result.rendered_content message.rendered_content = rendered_content message.rendered_content_version = markdown_version event["content"] = content @@ -5859,7 +5865,7 @@ def do_update_message( send_notification_to_old_thread: bool, send_notification_to_new_thread: bool, content: Optional[str], - rendered_content: Optional[str], + rendering_result: Optional[MessageRenderingResult], prior_mention_user_ids: Set[int], mention_data: Optional[MentionData] = None, ) -> int: @@ -5902,17 +5908,17 @@ def do_update_message( ums = UserMessage.objects.filter(message=target_message.id) if content is not None: - assert rendered_content is not None + assert rendering_result is not None # mention_data is required if there's a content edit. assert mention_data is not None # add data from group mentions to mentions_user_ids. - for group_id in target_message.mentions_user_group_ids: + for group_id in rendering_result.mentions_user_group_ids: members = mention_data.get_group_members(group_id) - target_message.mentions_user_ids.update(members) + rendering_result.mentions_user_ids.update(members) - update_user_message_flags(target_message, ums) + update_user_message_flags(rendering_result, ums) # One could imagine checking realm.allow_edit_history here and # modifying the events based on that setting, but doing so @@ -5930,16 +5936,20 @@ def do_update_message( "prev_rendered_content_version" ] = target_message.rendered_content_version target_message.content = content - target_message.rendered_content = rendered_content + target_message.rendered_content = rendering_result.rendered_content target_message.rendered_content_version = markdown_version event["content"] = content - event["rendered_content"] = rendered_content + event["rendered_content"] = rendering_result.rendered_content event["prev_rendered_content_version"] = target_message.rendered_content_version - event["is_me_message"] = Message.is_status_message(content, rendered_content) + event["is_me_message"] = Message.is_status_message( + content, rendering_result.rendered_content + ) # target_message.has_image and target_message.has_link will have been # already updated by Markdown rendering in the caller. - target_message.has_attachment = check_attachment_reference_change(target_message) + target_message.has_attachment = check_attachment_reference_change( + target_message, rendering_result + ) if target_message.is_stream_message(): if topic_name is not None: @@ -5968,7 +5978,7 @@ def do_update_message( event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"]) event["prior_mention_user_ids"] = list(prior_mention_user_ids) event["presence_idle_user_ids"] = filter_presence_idle_user_ids(info["active_user_ids"]) - if target_message.mentions_wildcard: + if rendering_result.mentions_wildcard: event["wildcard_mention_user_ids"] = list(info["wildcard_mention_user_ids"]) else: event["wildcard_mention_user_ids"] = [] @@ -5976,7 +5986,7 @@ def do_update_message( do_update_mobile_push_notification( target_message, prior_mention_user_ids, - target_message.mentions_user_ids, + rendering_result.mentions_user_ids, info["stream_push_user_ids"], ) @@ -7360,12 +7370,14 @@ def do_delete_old_unclaimed_attachments(weeks_ago: int) -> None: attachment.delete() -def check_attachment_reference_change(message: Message) -> bool: +def check_attachment_reference_change( + message: Message, rendering_result: MessageRenderingResult +) -> bool: # For a unsaved message edit (message.* has been updated, but not # saved to the database), adjusts Attachment data to correspond to # the new content. prev_attachments = {a.path_id for a in message.attachment_set.all()} - new_attachments = set(message.potential_attachment_path_ids) + new_attachments = set(rendering_result.potential_attachment_path_ids) if new_attachments == prev_attachments: return bool(prev_attachments) diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index ce8892acf7..23777b11ce 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -347,7 +347,7 @@ def fix_message_rendered_content( message_realm=realm, sent_by_bot=sent_by_bot, translate_emoticons=translate_emoticons, - ) + ).rendered_content message["rendered_content"] = rendered_content message["rendered_content_version"] = markdown_version diff --git a/zerver/lib/markdown/__init__.py b/zerver/lib/markdown/__init__.py index 664d30cbd6..20b6ea2c91 100644 --- a/zerver/lib/markdown/__init__.py +++ b/zerver/lib/markdown/__init__.py @@ -89,6 +89,18 @@ class LinkInfo(TypedDict): remove: Optional[Element] +@dataclass +class MessageRenderingResult: + rendered_content: str + mentions_wildcard: bool + mentions_user_ids: Set[int] + mentions_user_group_ids: Set[int] + alert_words: Set[str] + links_for_preview: Set[str] + user_ids_with_alert_words: Set[int] + potential_attachment_path_ids: List[str] + + DbData = Dict[str, Any] # Format version of the Markdown rendering; stored along with rendered @@ -1220,7 +1232,6 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): if self.md.zulip_message: self.md.zulip_message.has_link = len(found_urls) > 0 self.md.zulip_message.has_image = False # This is updated in self.add_a - self.md.zulip_message.potential_attachment_path_ids = [] for url in unique_urls: # Due to rewrite_local_links_to_relative, we need to @@ -1238,7 +1249,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): continue path_id = parsed_url.path[len("/user_uploads/") :] - self.md.zulip_message.potential_attachment_path_ids.append(path_id) + self.md.zulip_rendering_result.potential_attachment_path_ids.append(path_id) if len(found_urls) == 0: return @@ -1321,7 +1332,7 @@ class InlineInterestingLinkProcessor(markdown.treeprocessors.Treeprocessor): try: extracted_data = link_preview.link_embed_data_from_cache(url) except NotFoundInCache: - self.md.zulip_message.links_for_preview.add(url) + self.md.zulip_rendering_result.links_for_preview.add(url) continue if extracted_data: @@ -1828,11 +1839,11 @@ class UserMentionPattern(CompiledInlineProcessor): if wildcard: if not silent: - self.md.zulip_message.mentions_wildcard = True + self.md.zulip_rendering_result.mentions_wildcard = True user_id = "*" elif user: if not silent: - self.md.zulip_message.mentions_user_ids.add(user["id"]) + self.md.zulip_rendering_result.mentions_user_ids.add(user["id"]) name = user["full_name"] user_id = str(user["id"]) else: @@ -1864,7 +1875,7 @@ class UserGroupMentionPattern(CompiledInlineProcessor): user_group = db_data["mention_data"].get_user_group(name) if user_group: if not silent: - self.md.zulip_message.mentions_user_group_ids.add(user_group.id) + self.md.zulip_rendering_result.mentions_user_group_ids.add(user_group.id) name = user_group.name user_group_id = str(user_group.id) else: @@ -1994,7 +2005,7 @@ class AlertWordNotificationProcessor(markdown.preprocessors.Preprocessor): # # Our caller passes in the list of possible_words. We # don't do any special rendering; we just append the alert words - # we find to the set self.md.zulip_message.alert_words. + # we find to the set self.md.zulip_rendering_result.user_ids_with_alert_words. realm_alert_words_automaton = db_data["realm_alert_words_automaton"] @@ -2006,7 +2017,7 @@ class AlertWordNotificationProcessor(markdown.preprocessors.Preprocessor): if self.check_valid_start_position( content, end_index - len(original_value) ) and self.check_valid_end_position(content, end_index + 1): - self.md.zulip_message.user_ids_with_alert_words.update(user_ids) + self.md.zulip_rendering_result.user_ids_with_alert_words.update(user_ids) return lines @@ -2070,6 +2081,7 @@ class Markdown(markdown.Markdown): zulip_message: Optional[Message] zulip_realm: Optional[Realm] zulip_db_data: Optional[DbData] + zulip_rendering_result: Optional[MessageRenderingResult] image_preview_enabled: bool url_embed_preview_enabled: bool @@ -2387,7 +2399,7 @@ def do_convert( mention_data: Optional[MentionData] = None, email_gateway: bool = False, no_previews: bool = False, -) -> str: +) -> MessageRenderingResult: """Convert Markdown to HTML, with Zulip-specific settings and hacks.""" # This logic is a bit convoluted, but the overall goal is to support a range of use cases: # * Nothing is passed in other than content -> just run default options (e.g. for docs) @@ -2420,7 +2432,19 @@ def do_convert( _md_engine.reset() # Filters such as UserMentionPattern need a message. + rendering_result: MessageRenderingResult = MessageRenderingResult( + rendered_content="", + mentions_wildcard=False, + mentions_user_ids=set(), + mentions_user_group_ids=set(), + alert_words=set(), + links_for_preview=set(), + user_ids_with_alert_words=set(), + potential_attachment_path_ids=[], + ) + _md_engine.zulip_message = message + _md_engine.zulip_rendering_result = rendering_result _md_engine.zulip_realm = message_realm _md_engine.zulip_db_data = None # for now _md_engine.image_preview_enabled = image_preview_enabled(message, message_realm, no_previews) @@ -2464,17 +2488,17 @@ def do_convert( # extremely inefficient in corner cases) as well as user # errors (e.g. a linkifier that makes some syntax # infinite-loop). - rendered_content = timeout(5, lambda: _md_engine.convert(content)) + rendering_result.rendered_content = timeout(5, lambda: _md_engine.convert(content)) # Throw an exception if the content is huge; this protects the # rest of the codebase from any bugs where we end up rendering # something huge. MAX_MESSAGE_LENGTH = settings.MAX_MESSAGE_LENGTH - if len(rendered_content) > MAX_MESSAGE_LENGTH * 100: + if len(rendering_result.rendered_content) > MAX_MESSAGE_LENGTH * 100: raise MarkdownRenderingException( f"Rendered content exceeds {MAX_MESSAGE_LENGTH * 100} characters (message {logging_message_id})" ) - return rendered_content + return rendering_result except Exception: cleaned = privacy_clean_markdown(content) # NOTE: Don't change this message without also changing the @@ -2532,7 +2556,7 @@ def markdown_convert( mention_data: Optional[MentionData] = None, email_gateway: bool = False, no_previews: bool = False, -) -> str: +) -> MessageRenderingResult: markdown_stats_start() ret = do_convert( content, diff --git a/zerver/lib/message.py b/zerver/lib/message.py index 3abdb9b2c8..0d5dc2adb4 100644 --- a/zerver/lib/message.py +++ b/zerver/lib/message.py @@ -27,7 +27,7 @@ from zerver.lib.display_recipient import ( UserDisplayRecipient, bulk_fetch_display_recipients, ) -from zerver.lib.markdown import markdown_convert, topic_links +from zerver.lib.markdown import MessageRenderingResult, markdown_convert, topic_links from zerver.lib.markdown import version as markdown_version from zerver.lib.mention import MentionData from zerver.lib.request import JsonableError @@ -90,6 +90,7 @@ class UnreadMessagesResult(TypedDict): @dataclass class SendMessageRequest: message: Message + rendering_result: MessageRenderingResult stream: Optional[Stream] local_id: Optional[str] sender_queue_id: Optional[str] @@ -227,7 +228,10 @@ def message_to_dict_json(message: Message, realm_id: Optional[int] = None) -> by def save_message_rendered_content(message: Message, content: str) -> str: - rendered_content = render_markdown(message, content, realm=message.get_realm()) + rendering_result = render_markdown(message, content, realm=message.get_realm()) + rendered_content = None + if rendering_result is not None: + rendered_content = rendering_result.rendered_content message.rendered_content = rendered_content message.rendered_content_version = markdown_version message.save_rendered_content() @@ -810,7 +814,7 @@ def render_markdown( realm_alert_words_automaton: Optional[ahocorasick.Automaton] = None, mention_data: Optional[MentionData] = None, email_gateway: bool = False, -) -> str: +) -> MessageRenderingResult: """ This is basically just a wrapper for do_render_markdown. """ @@ -822,7 +826,7 @@ def render_markdown( sent_by_bot = sender.is_bot translate_emoticons = sender.translate_emoticons - rendered_content = do_render_markdown( + result = do_render_markdown( message=message, content=content, realm=realm, @@ -833,7 +837,7 @@ def render_markdown( email_gateway=email_gateway, ) - return rendered_content + return result def do_render_markdown( @@ -845,22 +849,9 @@ def do_render_markdown( realm_alert_words_automaton: Optional[ahocorasick.Automaton] = None, mention_data: Optional[MentionData] = None, email_gateway: bool = False, -) -> str: - """Return HTML for given Markdown. Markdown may add properties to the - message object such as `mentions_user_ids`, `mentions_user_group_ids`, and - `mentions_wildcard`. These are only on this Django object and are not - saved in the database. - """ - - message.mentions_wildcard = False - message.mentions_user_ids = set() - message.mentions_user_group_ids = set() - message.alert_words = set() - message.links_for_preview = set() - message.user_ids_with_alert_words = set() - +) -> MessageRenderingResult: # DO MAIN WORK HERE -- call markdown_convert to convert - rendered_content = markdown_convert( + rendering_result = markdown_convert( content, realm_alert_words_automaton=realm_alert_words_automaton, message=message, @@ -870,7 +861,7 @@ def do_render_markdown( mention_data=mention_data, email_gateway=email_gateway, ) - return rendered_content + return rendering_result def huddle_users(recipient_id: int) -> str: diff --git a/zerver/lib/realm_description.py b/zerver/lib/realm_description.py index da7013c606..67e9ba6d4c 100644 --- a/zerver/lib/realm_description.py +++ b/zerver/lib/realm_description.py @@ -11,7 +11,9 @@ from zerver.models import Realm @cache_with_key(realm_rendered_description_cache_key, timeout=3600 * 24 * 7) def get_realm_rendered_description(realm: Realm) -> str: realm_description_raw = realm.description or "The coolest place in the universe." - return markdown_convert(realm_description_raw, message_realm=realm, no_previews=True) + return markdown_convert( + realm_description_raw, message_realm=realm, no_previews=True + ).rendered_content @cache_with_key(realm_text_description_cache_key, timeout=3600 * 24 * 7) diff --git a/zerver/lib/streams.py b/zerver/lib/streams.py index 4063e35aaf..d945663e4a 100644 --- a/zerver/lib/streams.py +++ b/zerver/lib/streams.py @@ -75,7 +75,7 @@ def get_default_value_for_history_public_to_subscribers( def render_stream_description(text: str) -> str: - return markdown_convert(text, no_previews=True) + return markdown_convert(text, no_previews=True).rendered_content def send_stream_creation_event(stream: Stream, user_ids: List[int]) -> None: diff --git a/zerver/tests/test_custom_profile_data.py b/zerver/tests/test_custom_profile_data.py index db1fb0e71e..c2e0947d1d 100644 --- a/zerver/tests/test_custom_profile_data.py +++ b/zerver/tests/test_custom_profile_data.py @@ -617,7 +617,7 @@ class UpdateCustomProfileFieldTest(CustomProfileFieldTestCase): expected_rendered_value: Dict[Union[int, float, str, None], Union[str, None]] = {} for f in data: if f["field"].is_renderable(): - expected_rendered_value[f["id"]] = markdown_convert(f["value"]) + expected_rendered_value[f["id"]] = markdown_convert(f["value"]).rendered_content else: expected_rendered_value[f["id"]] = None diff --git a/zerver/tests/test_events.py b/zerver/tests/test_events.py index 006e745d09..8995617fb1 100644 --- a/zerver/tests/test_events.py +++ b/zerver/tests/test_events.py @@ -451,7 +451,7 @@ class NormalActionsTest(BaseAction): topic = "new_topic" propagate_mode = "change_all" content = "new content" - rendered_content = render_markdown(message, content) + rendering_result = render_markdown(message, content) prior_mention_user_ids: Set[int] = set() mention_data = MentionData( realm_id=self.user_profile.realm_id, @@ -468,7 +468,7 @@ class NormalActionsTest(BaseAction): False, False, content, - rendered_content, + rendering_result, prior_mention_user_ids, mention_data, ), @@ -482,10 +482,10 @@ class NormalActionsTest(BaseAction): has_new_stream_id=False, ) + content = "embed_content" + rendering_result = render_markdown(message, content) events = self.verify_action( - lambda: do_update_embedded_data( - self.user_profile, message, "embed_content", "

embed_content

" - ), + lambda: do_update_embedded_data(self.user_profile, message, content, rendering_result), state_change_expected=False, ) check_update_message_embedded("events[0]", events[0]) diff --git a/zerver/tests/test_markdown.py b/zerver/tests/test_markdown.py index 480a5b8dce..753f98e250 100644 --- a/zerver/tests/test_markdown.py +++ b/zerver/tests/test_markdown.py @@ -25,6 +25,7 @@ from zerver.lib.emoji import get_emoji_url from zerver.lib.exceptions import MarkdownRenderingException from zerver.lib.markdown import ( MarkdownListPreprocessor, + MessageRenderingResult, clear_state_for_testing, content_has_emoji_syntax, fetch_tweet_data, @@ -200,7 +201,7 @@ def markdown_convert_wrapper(content: str) -> str: return markdown_convert( content=content, message_realm=get_realm("zulip"), - ) + ).rendered_content class MarkdownMiscTest(ZulipTestCase): @@ -438,7 +439,8 @@ class MarkdownTest(ZulipTestCase): user_profile = self.example_user("othello") do_set_user_display_setting(user_profile, "translate_emoticons", True) msg = Message(sender=user_profile, sending_client=get_client("test")) - converted = render_markdown(msg, test["input"]) + rendering_result = render_markdown(msg, test["input"]) + converted = rendering_result.rendered_content else: converted = markdown_convert_wrapper(test["input"]) @@ -476,9 +478,9 @@ class MarkdownTest(ZulipTestCase): with self.settings(ENABLE_FILE_LINKS=False): realm = do_create_realm(string_id="file_links_test", name="file_links_test") maybe_update_markdown_engines(realm.id, False) - converted = markdown_convert(msg, message_realm=realm) self.assertEqual( - converted, "

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

" + markdown_convert(msg, message_realm=realm).rendered_content, + "

Check out this file file:///Volumes/myserver/Users/Shared/pi.py

", ) def test_inline_bitcoin(self) -> None: @@ -607,7 +609,7 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, with_preview) + self.assertEqual(converted.rendered_content, with_preview) realm = msg.get_realm() setattr(realm, "inline_image_preview", False) @@ -616,7 +618,7 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, without_preview) + self.assertEqual(converted.rendered_content, without_preview) @override_settings(THUMBNAIL_IMAGES=False, EXTERNAL_URI_SCHEME="https://") def test_external_image_preview_use_camo(self) -> None: @@ -655,21 +657,21 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) content = ">http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg\n\nAwesome!" expected = '
\n

http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg

\n
\n

Awesome!

' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) content = ">* http://cdn.wallpapersafari.com/13/6/16eVjx.jpeg\n\nAwesome!" expected = '
\n\n
\n

Awesome!

' sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_inline_image_preview_order(self) -> None: @@ -680,7 +682,7 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) content = "http://imaging.nikon.com/lineup/dslr/df/img/sample/img_01.jpg\n\n>http://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg\n\n* http://imaging.nikon.com/lineup/dslr/df/img/sample/img_03.jpg\n* https://www.google.com/images/srpr/logo4w.png" expected = '
\n

http://imaging.nikon.com/lineup/dslr/df/img/sample/img_02.jpg

\n
\n' @@ -688,7 +690,7 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) content = "Test 1\n[21136101110_1dde1c1a7e_o.jpg](/user_uploads/{realm_id}/6d/F1PX6u16JA2P-nK45PyxHIYZ/21136101110_1dde1c1a7e_o.jpg) \n\nNext image\n[IMG_20161116_023910.jpg](/user_uploads/{realm_id}/69/sh7L06e7uH7NaX6d5WFfVYQp/IMG_20161116_023910.jpg) \n\nAnother screenshot\n[Screenshot-from-2016-06-01-16-22-42.png](/user_uploads/{realm_id}/70/_aZmIEWaN1iUaxwkDjkO7bpj/Screenshot-from-2016-06-01-16-22-42.png)" content = content.format(realm_id=realm.id) @@ -697,7 +699,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=True) def test_corrected_image_source(self) -> None: @@ -708,7 +710,7 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) @override_settings(INLINE_IMAGE_PREVIEW=False) def test_image_preview_enabled(self) -> None: @@ -1183,14 +1185,14 @@ class MarkdownTest(ZulipTestCase): realm=realm, name="green_tick", deactivated=False ).get() self.assertEqual( - converted, + converted.rendered_content, "

{}

".format(emoji_img(":green_tick:", realm_emoji.file_name, realm.id)), ) # Deactivate realm emoji. do_remove_realm_emoji(realm, "green_tick") converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) - self.assertEqual(converted, "

:green_tick:

") + self.assertEqual(converted.rendered_content, "

:green_tick:

") def test_deactivated_realm_emoji(self) -> None: # Deactivate realm emoji. @@ -1199,7 +1201,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=self.example_user("hamlet")) converted = markdown_convert(":green_tick:", message_realm=realm, message=msg) - self.assertEqual(converted, "

:green_tick:

") + self.assertEqual(converted.rendered_content, "

:green_tick:

") def test_unicode_emoji(self) -> None: msg = "\u2615" # ☕ @@ -1224,7 +1226,7 @@ class MarkdownTest(ZulipTestCase): content = ":)" expected = "

:)

" converted = render_markdown(msg, content) - self.assertEqual(converted, expected) + self.assertEqual(converted.rendered_content, expected) def test_same_markup(self) -> None: msg = "\u2615" # ☕ @@ -1309,7 +1311,7 @@ class MarkdownTest(ZulipTestCase): converted_topic = topic_links(realm.id, msg.topic_name()) self.assertEqual( - converted, + converted.rendered_content, '

We should fix #224 and #115, but not issue#124 or #1124z or trac #15 today.

', ) self.assertEqual( @@ -1337,12 +1339,12 @@ class MarkdownTest(ZulipTestCase): converted = markdown_convert(content, message_realm=realm, message=msg) self.assertEqual( - converted, + converted.rendered_content, '

#ZUL-123 was fixed and code was deployed to production, also #zul-321 was deployed to staging

', ) def assert_conversion(content: str, should_have_converted: bool = True) -> None: - converted = markdown_convert(content, message_realm=realm, message=msg) + converted = markdown_convert(content, message_realm=realm, message=msg).rendered_content converted_topic = topic_links(realm.id, content) if should_have_converted: self.assertTrue("https://trac.example.com" in converted) @@ -1440,7 +1442,7 @@ class MarkdownTest(ZulipTestCase): # The second linkifier (which was saved later) was ignored as the content was marked AtomicString after first conversion. # There was no easy way to support parsing both linkifiers and not run into an infinite loop, hence the second linkifier is ignored. self.assertEqual( - converted, + converted.rendered_content, '

We should fix ABC-123 or trac ABC-123 today.

', ) # Both the links should be generated in topics. @@ -1506,28 +1508,28 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=user_profile, sending_client=get_client("test")) content = "/me makes a list\n* one\n* two" - rendered_content = render_markdown(msg, content) + rendering_result = render_markdown(msg, content) self.assertEqual( - rendered_content, + rendering_result.rendered_content, "

/me makes a list

\n", ) - self.assertTrue(Message.is_status_message(content, rendered_content)) + self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) content = "/me takes a walk" - rendered_content = render_markdown(msg, content) + rendering_result = render_markdown(msg, content) self.assertEqual( - rendered_content, + rendering_result.rendered_content, "

/me takes a walk

", ) - self.assertTrue(Message.is_status_message(content, rendered_content)) + self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) content = "/me writes a second line\nline" - rendered_content = render_markdown(msg, content) + rendering_result = render_markdown(msg, content) self.assertEqual( - rendered_content, + rendering_result.rendered_content, "

/me writes a second line
\nline

", ) - self.assertTrue(Message.is_status_message(content, rendered_content)) + self.assertTrue(Message.is_status_message(content, rendering_result.rendered_content)) def test_alert_words(self) -> None: user_profile = self.example_user("othello") @@ -1535,19 +1537,25 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = "We have an ALERTWORD day today!" - self.assertEqual(render(msg, content), "

We have an ALERTWORD day today!

") - self.assertEqual(msg.user_ids_with_alert_words, {user_profile.id}) + rendering_result = render(msg, content) + self.assertEqual( + rendering_result.rendered_content, "

We have an ALERTWORD day today!

" + ) + self.assertEqual(rendering_result.user_ids_with_alert_words, {user_profile.id}) msg = Message(sender=user_profile, sending_client=get_client("test")) content = "We have a NOTHINGWORD day today!" - self.assertEqual(render(msg, content), "

We have a NOTHINGWORD day today!

") - self.assertEqual(msg.user_ids_with_alert_words, set()) + rendering_result = render(msg, content) + self.assertEqual( + rendering_result.rendered_content, "

We have a NOTHINGWORD day today!

" + ) + self.assertEqual(rendering_result.user_ids_with_alert_words, set()) def test_alert_words_returns_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { @@ -1567,13 +1575,13 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = "hello how is this possible how are you doing today" - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, @@ -1582,7 +1590,7 @@ class MarkdownTest(ZulipTestCase): user_profiles["othello"].id, } # All users except aaron have their alert word appear in the message content - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_1(self) -> None: alert_words_for_users: Dict[str, List[str]] = { @@ -1601,7 +1609,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) @@ -1611,7 +1619,7 @@ class MarkdownTest(ZulipTestCase): to test out how Markdown convert this into something line ending split array and this is a new line last""" - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = { user_profiles["hamlet"].id, user_profiles["cordelia"].id, @@ -1620,7 +1628,7 @@ class MarkdownTest(ZulipTestCase): user_profiles["othello"].id, } # All users have their alert word appear in the message content - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_user_ids_with_alert_words_in_french(self) -> None: alert_words_for_users: Dict[str, List[str]] = { @@ -1639,7 +1647,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) @@ -1648,10 +1656,10 @@ class MarkdownTest(ZulipTestCase): bonjour est (énormément) ce a quoi ressemble le français et j'espère qu'il n'y n' réglementaire a pas de mots d'alerte dans ce texte français """ - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id, user_profiles["cordelia"].id} # Only hamlet and cordelia have their alert-words appear in the message content - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_returns_empty_user_ids_with_alert_words(self) -> None: alert_words_for_users: Dict[str, List[str]] = { @@ -1671,7 +1679,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) @@ -1680,10 +1688,10 @@ class MarkdownTest(ZulipTestCase): This is to test that the no user_ids who have alrert wourldword is participating in sending of the message """ - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = set() # None of the users have their alert-words appear in the message content - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def get_mock_alert_words(self, num_words: int, word_length: int) -> List[str]: alert_words = ["x" * word_length] * num_words # type List[str] @@ -1705,15 +1713,15 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) content = """This is to test a empty alert words i.e. no user has any alert-words set""" - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = set() - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_alert_words_retuns_user_ids_with_alert_words_with_huge_alert_words(self) -> None: @@ -1732,7 +1740,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) realm_alert_words_automaton = get_alert_word_automaton(sender_user_profile.realm) - def render(msg: Message, content: str) -> str: + def render(msg: Message, content: str) -> MessageRenderingResult: return render_markdown( msg, content, realm_alert_words_automaton=realm_alert_words_automaton ) @@ -1744,10 +1752,10 @@ class MarkdownTest(ZulipTestCase): etc.). I was talking abou the issue124 on github. Then the third line: print random.randint(1,101) will automatically select a random integer between 1 and 100 for you. The process is fairly simple """ - render(msg, content) + rendering_result = render(msg, content) expected_user_ids: Set[int] = {user_profiles["hamlet"].id} # Only hamlet has alert-word 'issue124' present in the message content - self.assertEqual(msg.user_ids_with_alert_words, expected_user_ids) + self.assertEqual(rendering_result.user_ids_with_alert_words, expected_user_ids) def test_default_code_block_language(self) -> None: realm = get_realm("zulip") @@ -1810,69 +1818,76 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@**all** test" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@all" " test

", ) - self.assertTrue(msg.mentions_wildcard) + self.assertTrue(rendering_result.mentions_wildcard) def test_mention_everyone(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@**everyone** test" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@everyone" " test

", ) - self.assertTrue(msg.mentions_wildcard) + self.assertTrue(rendering_result.mentions_wildcard) def test_mention_stream(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@**stream** test" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@stream" " test

", ) - self.assertTrue(msg.mentions_wildcard) + self.assertTrue(rendering_result.mentions_wildcard) def test_mention_at_wildcard(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@all test" - self.assertEqual(render_markdown(msg, content), "

@all test

") - self.assertFalse(msg.mentions_wildcard) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, "

@all test

") + self.assertFalse(rendering_result.mentions_wildcard) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_mention_at_everyone(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@everyone test" - self.assertEqual(render_markdown(msg, content), "

@everyone test

") - self.assertFalse(msg.mentions_wildcard) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, "

@everyone test

") + self.assertFalse(rendering_result.mentions_wildcard) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_mention_word_starting_with_at_wildcard(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "test @alleycat.com test" - self.assertEqual(render_markdown(msg, content), "

test @alleycat.com test

") - self.assertFalse(msg.mentions_wildcard) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, "

test @alleycat.com test

") + self.assertFalse(rendering_result.mentions_wildcard) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_mention_at_normal_user(self) -> None: user_profile = self.example_user("othello") msg = Message(sender=user_profile, sending_client=get_client("test")) content = "@aaron test" - self.assertEqual(render_markdown(msg, content), "

@aaron test

") - self.assertFalse(msg.mentions_wildcard) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, "

@aaron test

") + self.assertFalse(rendering_result.mentions_wildcard) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_mention_single(self) -> None: sender_user_profile = self.example_user("othello") @@ -1881,18 +1896,20 @@ class MarkdownTest(ZulipTestCase): user_id = user_profile.id content = "@**King Hamlet**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) content = f"@**|{user_id}**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) def test_mention_silent(self) -> None: sender_user_profile = self.example_user("othello") @@ -1901,13 +1918,14 @@ class MarkdownTest(ZulipTestCase): user_id = user_profile.id content = "@_**King Hamlet**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_silent_wildcard_mention(self) -> None: user_profile = self.example_user("othello") @@ -1916,11 +1934,12 @@ class MarkdownTest(ZulipTestCase): wildcards = ["all", "everyone", "stream"] for wildcard in wildcards: content = f"@_**{wildcard}**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, f'

{wildcard}

', ) - self.assertFalse(msg.mentions_wildcard) + self.assertFalse(rendering_result.mentions_wildcard) def test_mention_invalid_followed_by_valid(self) -> None: sender_user_profile = self.example_user("othello") @@ -1929,13 +1948,14 @@ class MarkdownTest(ZulipTestCase): user_id = user_profile.id content = "@**Invalid user** and @**King Hamlet**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

@Invalid user and ' "@King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) def test_invalid_mention_not_uses_valid_mention_data(self) -> None: sender_user_profile = self.example_user("othello") @@ -1948,12 +1968,13 @@ class MarkdownTest(ZulipTestCase): # to use that data for creating a valid mention. content = f"@**King Hamlet|10** and @**aaron|{hamlet.id}**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, f'

' f"@King Hamlet and @aaron|{hamlet.id}

", ) - self.assertEqual(msg.mentions_user_ids, {hamlet.id}) + self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id}) def test_silent_mention_invalid_followed_by_valid(self) -> None: sender_user_profile = self.example_user("othello") @@ -1962,23 +1983,25 @@ class MarkdownTest(ZulipTestCase): user_id = user_profile.id content = "@_**Invalid user** and @_**King Hamlet**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

@_Invalid user and ' "King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) content = f"@_**|123456789** and @_**|{user_id}**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "

@_|123456789 and " '' "King Hamlet

", ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_possible_mentions(self) -> None: def assert_mentions(content: str, names: Set[str], has_wildcards: bool = False) -> None: @@ -2005,8 +2028,9 @@ class MarkdownTest(ZulipTestCase): content = "@**King Hamlet** and @**Cordelia, Lear's daughter**, check this out" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "

" '@King Hamlet and ' @@ -2014,7 +2038,7 @@ class MarkdownTest(ZulipTestCase): f'data-user-id="{cordelia.id}">@Cordelia, Lear\'s daughter, ' "check this out

", ) - self.assertEqual(msg.mentions_user_ids, {hamlet.id, cordelia.id}) + self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id, cordelia.id}) def test_mention_in_quotes(self) -> None: othello = self.example_user("othello") @@ -2023,8 +2047,9 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=othello, sending_client=get_client("test")) content = "> @**King Hamlet** and @**Othello, the Moor of Venice**\n\n @**King Hamlet** and @**Cordelia, Lear's daughter**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "
\n

" f'King Hamlet' " and " @@ -2036,7 +2061,7 @@ class MarkdownTest(ZulipTestCase): f'@Cordelia, Lear\'s daughter' "

", ) - self.assertEqual(msg.mentions_user_ids, {hamlet.id, cordelia.id}) + self.assertEqual(rendering_result.mentions_user_ids, {hamlet.id, cordelia.id}) # Both fenced quote and > quote should be identical for both silent and regular syntax. expected = ( @@ -2045,21 +2070,25 @@ class MarkdownTest(ZulipTestCase): "

\n
" ) content = "```quote\n@**King Hamlet**\n```" - self.assertEqual(render_markdown(msg, content), expected) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertEqual(rendering_result.mentions_user_ids, set()) content = "> @**King Hamlet**" - self.assertEqual(render_markdown(msg, content), expected) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertEqual(rendering_result.mentions_user_ids, set()) content = "```quote\n@_**King Hamlet**\n```" - self.assertEqual(render_markdown(msg, content), expected) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertEqual(rendering_result.mentions_user_ids, set()) content = "> @_**King Hamlet**" - self.assertEqual(render_markdown(msg, content), expected) - self.assertEqual(msg.mentions_user_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_wildcard_mention_in_quotes(self) -> None: user_profile = self.example_user("othello") - msg = Message(sender=user_profile, sending_client=get_client("test")) + message = Message(sender=user_profile, sending_client=get_client("test")) def assert_silent_mention(content: str, wildcard: str) -> None: expected = ( @@ -2067,8 +2096,9 @@ class MarkdownTest(ZulipTestCase): f'{wildcard}' "

\n" ) - self.assertEqual(render_markdown(msg, content), expected) - self.assertFalse(msg.mentions_wildcard) + rendering_result = render_markdown(message, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertFalse(rendering_result.mentions_wildcard) wildcards = ["all", "everyone", "stream"] for wildcard in wildcards: @@ -2096,8 +2126,9 @@ class MarkdownTest(ZulipTestCase): content = f"@**Mark Twin|{twin1.id}**, @**Mark Twin|{twin2.id}** and @**Cordelia, Lear's daughter**, hi." + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "

" '@Mark Twin, ' @@ -2107,17 +2138,18 @@ class MarkdownTest(ZulipTestCase): f'data-user-id="{cordelia.id}">@Cordelia, Lear\'s daughter, ' "hi.

", ) - self.assertEqual(msg.mentions_user_ids, {twin1.id, twin2.id, cordelia.id}) + self.assertEqual(rendering_result.mentions_user_ids, {twin1.id, twin2.id, cordelia.id}) def test_mention_invalid(self) -> None: sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "Hey @**Nonexistent User**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), "

Hey @Nonexistent User

" + rendering_result.rendered_content, "

Hey @Nonexistent User

" ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_user_mention_atomic_string(self) -> None: sender_user_profile = self.example_user("othello") @@ -2141,21 +2173,23 @@ class MarkdownTest(ZulipTestCase): full_name="Atomic #123", ) content = "@**Atomic #123**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@Atomic #123

", ) - self.assertEqual(msg.mentions_user_ids, {test_user.id}) + self.assertEqual(rendering_result.mentions_user_ids, {test_user.id}) content = "@_**Atomic #123**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "Atomic #123

", ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) def create_user_group_for_test(self, user_group_name: str) -> UserGroup: othello = self.example_user("othello") @@ -2169,8 +2203,9 @@ class MarkdownTest(ZulipTestCase): user_group = self.create_user_group_for_test("support") content = "@**King Hamlet** @*support*" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@King Hamlet " @@ -2178,8 +2213,8 @@ class MarkdownTest(ZulipTestCase): f'data-user-group-id="{user_group.id}">' "@support

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) - self.assertEqual(msg.mentions_user_group_ids, {user_group.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id}) def test_invalid_user_group_followed_by_valid_mention_single(self) -> None: sender_user_profile = self.example_user("othello") @@ -2189,8 +2224,9 @@ class MarkdownTest(ZulipTestCase): user_group = self.create_user_group_for_test("support") content = "@**King Hamlet** @*Invalid user group* @*support*" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@King Hamlet " @@ -2199,8 +2235,8 @@ class MarkdownTest(ZulipTestCase): f'data-user-group-id="{user_group.id}">' "@support

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) - self.assertEqual(msg.mentions_user_group_ids, {user_group.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id}) def test_user_group_mention_atomic_string(self) -> None: sender_user_profile = self.example_user("othello") @@ -2222,8 +2258,9 @@ class MarkdownTest(ZulipTestCase): user_group = self.create_user_group_for_test("support #123") content = "@**King Hamlet** @*support #123*" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, '

' "@King Hamlet " @@ -2231,8 +2268,8 @@ class MarkdownTest(ZulipTestCase): f'data-user-group-id="{user_group.id}">' "@support #123

", ) - self.assertEqual(msg.mentions_user_ids, {user_profile.id}) - self.assertEqual(msg.mentions_user_group_ids, {user_group.id}) + self.assertEqual(rendering_result.mentions_user_ids, {user_profile.id}) + self.assertEqual(rendering_result.mentions_user_group_ids, {user_group.id}) def test_possible_user_group_mentions(self) -> None: def assert_mentions(content: str, names: Set[str]) -> None: @@ -2261,8 +2298,9 @@ class MarkdownTest(ZulipTestCase): backend = self.create_user_group_for_test("backend") content = "@*support* and @*backend*, check this out" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "

" '' @@ -2275,7 +2313,7 @@ class MarkdownTest(ZulipTestCase): "

", ) - self.assertEqual(msg.mentions_user_group_ids, {support.id, backend.id}) + self.assertEqual(rendering_result.mentions_user_group_ids, {support.id, backend.id}) def test_user_group_mention_edit(self) -> None: sender_user_profile = self.example_user("hamlet") @@ -2317,8 +2355,11 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "Hey @*Nonexistent group*" - self.assertEqual(render_markdown(msg, content), "

Hey @Nonexistent group

") - self.assertEqual(msg.mentions_user_group_ids, set()) + rendering_result = render_markdown(msg, content) + self.assertEqual( + rendering_result.rendered_content, "

Hey @Nonexistent group

" + ) + self.assertEqual(rendering_result.mentions_user_group_ids, set()) def test_user_group_silent_mention(self) -> None: sender_user_profile = self.example_user("othello") @@ -2326,18 +2367,19 @@ class MarkdownTest(ZulipTestCase): support = self.create_user_group_for_test("support") content = "We'll add you to @_*support* user group." + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), + rendering_result.rendered_content, "

We'll add you to " f'support' " user group.

", ) - self.assertEqual(msg.mentions_user_group_ids, set()) + self.assertEqual(rendering_result.mentions_user_group_ids, set()) def test_user_group_mention_in_quotes(self) -> None: user_profile = self.example_user("othello") - msg = Message(sender=user_profile, sending_client=get_client("test")) + message = Message(sender=user_profile, sending_client=get_client("test")) backend = self.create_user_group_for_test("backend") def assert_silent_mention(content: str) -> None: @@ -2346,8 +2388,9 @@ class MarkdownTest(ZulipTestCase): f'backend' "

\n" ) - self.assertEqual(render_markdown(msg, content), expected) - self.assertEqual(msg.mentions_user_group_ids, set()) + rendering_result = render_markdown(message, content) + self.assertEqual(rendering_result.rendered_content, expected) + self.assertEqual(rendering_result.mentions_user_group_ids, set()) assert_silent_mention("> @*backend*") assert_silent_mention("> @_*backend*") @@ -2360,7 +2403,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "#**Denmark**" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#{d.name}

'.format( d=denmark, ), @@ -2372,7 +2415,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "#**Invalid** and #**Denmark**" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#Invalid and #{d.name}

'.format( d=denmark, ), @@ -2386,7 +2429,7 @@ class MarkdownTest(ZulipTestCase): scotland = get_stream("Scotland", realm) content = "Look to #**Denmark** and #**Scotland**, there something" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, "

Look to " '#{s.name}

'.format( s=case_sens, ), @@ -2419,7 +2462,9 @@ class MarkdownTest(ZulipTestCase): sender_user_profile = self.example_user("othello") msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "#**casesens**" - self.assertEqual(render_markdown(msg, content), "

#casesens

") + self.assertEqual( + render_markdown(msg, content).rendered_content, "

#casesens

" + ) def test_topic_single(self) -> None: denmark = get_stream("Denmark", get_realm("zulip")) @@ -2427,7 +2472,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "#**Denmark>some topic**" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#{d.name} > some topic

'.format( d=denmark, ), @@ -2451,7 +2496,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "#**Denmark>#1234**" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#{d.name} > #1234

'.format( d=denmark, ), @@ -2464,7 +2509,7 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "This has two links: #**Denmark>some topic** and #**Scotland>other topic**." self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, "

This has two links: " '' @@ -2495,7 +2540,7 @@ class MarkdownTest(ZulipTestCase): quoted_name = ".D0.BF.D1.80.D0.B8.D0.B2.D0.B5.D1.82" href = f"/#narrow/stream/{uni.id}-{quoted_name}" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#{s.name}

'.format( s=uni, href=href, @@ -2521,7 +2566,7 @@ class MarkdownTest(ZulipTestCase): content = "#**Stream #1234**" href = f"/#narrow/stream/{stream.id}-Stream-.231234" self.assertEqual( - render_markdown(msg, content), + render_markdown(msg, content).rendered_content, '

#{s.name}

'.format( s=stream, href=href, @@ -2533,10 +2578,11 @@ class MarkdownTest(ZulipTestCase): msg = Message(sender=sender_user_profile, sending_client=get_client("test")) content = "There #**Nonexistentstream**" + rendering_result = render_markdown(msg, content) self.assertEqual( - render_markdown(msg, content), "

There #Nonexistentstream

" + rendering_result.rendered_content, "

There #Nonexistentstream

" ) - self.assertEqual(msg.mentions_user_ids, set()) + self.assertEqual(rendering_result.mentions_user_ids, set()) def test_image_preview_title(self) -> None: msg = "[My favorite image](https://example.com/testimage.png)" @@ -2563,19 +2609,19 @@ class MarkdownTest(ZulipTestCase): message = Message(sending_client=client, sender=self.mit_user("sipbtest")) converted = markdown_convert(msg, message_realm=realm, message=message) self.assertEqual( - converted, + converted.rendered_content, "

**test**

", ) msg = "* test" converted = markdown_convert(msg, message_realm=realm, message=message) self.assertEqual( - converted, + converted.rendered_content, "

* test

", ) msg = "https://lists.debian.org/debian-ctte/2014/02/msg00173.html" converted = markdown_convert(msg, message_realm=realm, message=message) self.assertEqual( - converted, + converted.rendered_content, '

https://lists.debian.org/debian-ctte/2014/02/msg00173.html

', ) @@ -2604,12 +2650,12 @@ class MarkdownTest(ZulipTestCase): string_id="code_block_processor_test", name="code_block_processor_test" ) maybe_update_markdown_engines(realm.id, True) - converted = markdown_convert(msg, message_realm=realm, email_gateway=True) + rendering_result = markdown_convert(msg, message_realm=realm, email_gateway=True) expected_output = ( "

Hello,

\n" + "

I am writing this message to test something. I am writing this message to test something.

" ) - self.assertEqual(converted, expected_output) + self.assertEqual(rendering_result.rendered_content, expected_output) def test_normal_link(self) -> None: realm = get_realm("zulip") @@ -2618,7 +2664,7 @@ class MarkdownTest(ZulipTestCase): msg = "http://example.com/#settings/" self.assertEqual( - markdown_convert(msg, message_realm=realm, message=message), + markdown_convert(msg, message_realm=realm, message=message).rendered_content, '

http://example.com/#settings/

', ) @@ -2629,7 +2675,7 @@ class MarkdownTest(ZulipTestCase): msg = "http://zulip.testserver/#narrow/stream/999-hello" self.assertEqual( - markdown_convert(msg, message_realm=realm, message=message), + markdown_convert(msg, message_realm=realm, message=message).rendered_content, '

http://zulip.testserver/#narrow/stream/999-hello

', ) @@ -2640,7 +2686,7 @@ class MarkdownTest(ZulipTestCase): msg = "http://zulip.testserver/#streams/all" self.assertEqual( - markdown_convert(msg, message_realm=realm, message=message), + markdown_convert(msg, message_realm=realm, message=message).rendered_content, '

http://zulip.testserver/#streams/all

', ) @@ -2651,7 +2697,7 @@ class MarkdownTest(ZulipTestCase): msg = "[hello](http://zulip.testserver/#narrow/stream/999-hello)" self.assertEqual( - markdown_convert(msg, message_realm=realm, message=message), + markdown_convert(msg, message_realm=realm, message=message).rendered_content, '

hello

', ) diff --git a/zerver/tests/test_message_edit.py b/zerver/tests/test_message_edit.py index b517baf730..bcf2b1f982 100644 --- a/zerver/tests/test_message_edit.py +++ b/zerver/tests/test_message_edit.py @@ -951,7 +951,7 @@ class EditMessageTest(EditMessageTestCase): send_notification_to_old_thread=False, send_notification_to_new_thread=False, content=None, - rendered_content=None, + rendering_result=None, prior_mention_user_ids=set(), mention_data=None, ) diff --git a/zerver/tests/test_message_fetch.py b/zerver/tests/test_message_fetch.py index 8e797f2e58..30ba2b2abe 100644 --- a/zerver/tests/test_message_fetch.py +++ b/zerver/tests/test_message_fetch.py @@ -3711,7 +3711,7 @@ class MessageHasKeywordsTest(ZulipTestCase): def update_message(self, msg: Message, content: str) -> None: hamlet = self.example_user("hamlet") realm_id = hamlet.realm.id - rendered_content = render_markdown(msg, content) + rendering_result = render_markdown(msg, content) mention_data = MentionData(realm_id, content) do_update_message( hamlet, @@ -3722,7 +3722,7 @@ class MessageHasKeywordsTest(ZulipTestCase): False, False, content, - rendered_content, + rendering_result, set(), mention_data=mention_data, ) diff --git a/zerver/views/message_send.py b/zerver/views/message_send.py index d57eabc76d..73e7ceb36e 100644 --- a/zerver/views/message_send.py +++ b/zerver/views/message_send.py @@ -325,5 +325,5 @@ def render_message_backend( message.content = content message.sending_client = request.client - rendered_content = render_markdown(message, content, realm=user_profile.realm) - return json_success({"rendered": rendered_content}) + rendering_result = render_markdown(message, content, realm=user_profile.realm) + return json_success({"rendered": rendering_result.rendered_content}) diff --git a/zerver/worker/queue_processors.py b/zerver/worker/queue_processors.py index b0ebac6fa5..c60f001c0e 100644 --- a/zerver/worker/queue_processors.py +++ b/zerver/worker/queue_processors.py @@ -757,10 +757,10 @@ class FetchLinksEmbedData(QueueProcessingWorker): realm = Realm.objects.get(id=event["message_realm_id"]) # If rendering fails, the called code will raise a JsonableError. - rendered_content = render_incoming_message( + rendering_result = render_incoming_message( message, message.content, message_user_ids, realm ) - do_update_embedded_data(message.sender, message, message.content, rendered_content) + do_update_embedded_data(message.sender, message, message.content, rendering_result) @assign_queue("outgoing_webhooks")