linkifier: Support URL templates for linkifiers.

This swaps out url_format_string from all of our APIs and replaces it
with url_template. Note that the documentation changes in the following
commits  will be squashed with this commit.

We change the "url_format" key to "url_template" for the
realm_linkifiers events in event_schema, along with updating
LinkifierDict. "url_template" is the name chosen to normalize
mixed usages of "url_format_string" and "url_format" throughout
the backend.

The markdown processor is updated to stop handling the format string
interpolation and delegate the task template expansion to the uri_template
library instead.

This change affects many test cases. We mostly just replace "%(name)s"
with "{name}", "url_format_string" with "url_template" to make sure that
they still pass. There are some test cases dedicated for testing "%"
escaping, which aren't relevant anymore and are subject to removal.
But for now we keep most of them as-is, and make sure that "%" is always
escaped since we do not use it for variable substitution any more.

Since url_format_string is not populated anymore, a migration is created
to remove this field entirely, and make url_template non-nullable since
we will always populate it. Note that it is possible to have
url_template being null after migration 0422 and before 0424, but
in practice, url_template will not be None after backfilling and the
backend now is always setting url_template.

With the removal of url_format_string, RealmFilter model will now be cleaned
with URL template checks, and the old checks for escapes are removed.

We also modified RealmFilter.clean to skip the validation when the
url_template is invalid. This avoids raising mulitple ValidationError's
when calling full_clean on a linkifier. But we might eventually want to
have a more centric approach to data validation instead of having
the same validation in both the clean method and the validator.

Fixes #23124.

Signed-off-by: Zixuan James Li <p359101898@gmail.com>

zerver/tornado/django_api.py

@@ -83,6 +83,7 @@ def request_event_queue(
     stream_typing_notifications: bool = False,
     user_settings_object: bool = False,
     pronouns_field_type_supported: bool = True,
+    linkifier_url_template: bool = False,
 ) -> Optional[str]:
     if not settings.USING_TORNADO:
         return None
@@ -104,6 +105,7 @@ def request_event_queue(
         "stream_typing_notifications": orjson.dumps(stream_typing_notifications),
         "user_settings_object": orjson.dumps(user_settings_object),
         "pronouns_field_type_supported": orjson.dumps(pronouns_field_type_supported),
+        "linkifier_url_template": orjson.dumps(linkifier_url_template),
     }
 
     if event_types is not None:

zerver/tornado/event_queue.py

@@ -96,6 +96,7 @@ class ClientDescriptor:
         stream_typing_notifications: bool = False,
         user_settings_object: bool = False,
         pronouns_field_type_supported: bool = True,
+        linkifier_url_template: bool = False,
     ) -> None:
         # These objects are serialized on shutdown and restored on restart.
         # If fields are added or semantics are changed, temporary code must be
@@ -120,6 +121,7 @@ class ClientDescriptor:
         self.stream_typing_notifications = stream_typing_notifications
         self.user_settings_object = user_settings_object
         self.pronouns_field_type_supported = pronouns_field_type_supported
+        self.linkifier_url_template = linkifier_url_template
 
         # Default for lifespan_secs is DEFAULT_EVENT_QUEUE_TIMEOUT_SECS;
         # but users can set it as high as MAX_QUEUE_TIMEOUT_SECS.
@@ -148,6 +150,7 @@ class ClientDescriptor:
             stream_typing_notifications=self.stream_typing_notifications,
             user_settings_object=self.user_settings_object,
             pronouns_field_type_supported=self.pronouns_field_type_supported,
+            linkifier_url_template=self.linkifier_url_template,
         )
 
     def __repr__(self) -> str:
@@ -181,6 +184,7 @@ class ClientDescriptor:
             d.get("stream_typing_notifications", False),
             d.get("user_settings_object", False),
             d.get("pronouns_field_type_supported", True),
+            d.get("linkifier_url_template", False),
         )
         ret.last_connection_time = d["last_connection_time"]
         return ret

zerver/tornado/views.py

@@ -156,6 +156,9 @@ def get_events_backend(
     pronouns_field_type_supported: bool = REQ(
         default=True, json_validator=check_bool, intentionally_undocumented=True
     ),
+    linkifier_url_template: bool = REQ(
+        default=False, json_validator=check_bool, intentionally_undocumented=True
+    ),
 ) -> HttpResponse:
     if all_public_streams and not user_profile.can_access_public_streams():
         raise JsonableError(_("User not authorized for this query"))
@@ -188,6 +191,7 @@ def get_events_backend(
             stream_typing_notifications=stream_typing_notifications,
             user_settings_object=user_settings_object,
             pronouns_field_type_supported=pronouns_field_type_supported,
+            linkifier_url_template=linkifier_url_template,
         )
 
     result = in_tornado_thread(fetch_events)(