emails: Inline CSS in emails in build_email.

Previously, we had an architecture where CSS inlining for emails was
done at provision time in inline_email_css.py. This was necessary
because the library we were using for this, Premailer, was extremely
slow, and doing the inlining for every outgoing email would have been
prohibitively expensive.

Now that we've migrated to a more modern library that inlines the
small amount of CSS we have into emails nearly instantly, we are able
to remove the complex architecture built to work around Premailer
being slow and just do the CSS inlining as the final step in sending
each individual email.

This has several significant benefits:

* Removes a fiddly provisioning step that made the edit/refresh cycle
  for modifying email templates confusing; there's no longer a CSS
  inlining step that, if you forget to do it, results in your testing a
  stale variant of the email templates.
* Fixes internationalization problems related to translators working
  with pre-CSS-inlined emails, and then Django trying to apply the
  translators to the post-CSS-inlined version.
* Makes the send_custom_email pipeline simpler and easier to improve.

Signed-off-by: Daniil Fadeev <fadeevd@zulip.com>

zerver/lib/send_email.py

@@ -11,6 +11,7 @@ from email.utils import formataddr, parseaddr
 from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import backoff
+import css_inline
 import orjson
 from django.conf import settings
 from django.core.mail import EmailMultiAlternatives, get_connection
@@ -21,18 +22,19 @@ from django.core.management import CommandError
 from django.db import transaction
 from django.http import HttpRequest
 from django.template import loader
-from django.template.exceptions import TemplateDoesNotExist
 from django.utils.timezone import now as timezone_now
 from django.utils.translation import gettext as _
 from django.utils.translation import override as override_language
 
 from confirmation.models import generate_key, one_click_unsubscribe_link
-from scripts.setup.inline_email_css import inline_template
 from zerver.lib.logging_util import log_to_file
 from zerver.models import EMAIL_TYPES, Realm, ScheduledEmail, UserProfile, get_user_profile_by_id
 from zproject.email_backends import EmailLogBackEnd, get_forward_address
 
 MAX_CONNECTION_TRIES = 3
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../")
+EMAIL_TEMPLATES_PATH = os.path.join(ZULIP_PATH, "templates", "zerver", "emails")
+CSS_SOURCE_PATH = os.path.join(EMAIL_TEMPLATES_PATH, "email.css")
 
 ## Logging setup ##
 
@@ -40,6 +42,12 @@ logger = logging.getLogger("zulip.send_email")
 log_to_file(logger, settings.EMAIL_LOG_PATH)
 
 
+def get_inliner_instance() -> css_inline.CSSInliner:
+    with open(CSS_SOURCE_PATH) as file:
+        content = file.read()
+    return css_inline.CSSInliner(extra_css=content)
+
+
 class FromAddress:
     SUPPORT = parseaddr(settings.ZULIP_ADMINISTRATOR)[1]
     NOREPLY = parseaddr(settings.NOREPLY_EMAIL_ADDRESS)[1]
@@ -124,6 +132,10 @@ def build_email(
         "physical_address": settings.PHYSICAL_ADDRESS,
     }
 
+    def get_inlined_template(template: str) -> str:
+        inliner = get_inliner_instance()
+        return inliner.inline(template)
+
     def render_templates() -> Tuple[str, str, str]:
         email_subject = (
             loader.render_to_string(
@@ -136,14 +148,8 @@ def build_email(
             template_prefix + ".txt", context=context, using="Jinja2_plaintext"
         )
 
-        try:
-            html_message = loader.render_to_string(template_prefix + ".html", context)
-        except TemplateDoesNotExist:
-            emails_dir = os.path.dirname(template_prefix)
-            template = os.path.basename(template_prefix)
-            compiled_template_prefix = os.path.join(emails_dir, "compiled", template)
-            html_message = loader.render_to_string(compiled_template_prefix + ".html", context)
-        return (html_message, message, email_subject)
+        html_message = loader.render_to_string(template_prefix + ".html", context)
+        return (get_inlined_template(html_message), message, email_subject)
 
     # The i18n story for emails is a bit complicated.  For emails
     # going to a single user, we want to use the language that user
@@ -518,13 +524,12 @@ def send_custom_email(
         parsed_email_template = Parser(policy=default).parsestr(text)
         email_template_hash = hashlib.sha256(text.encode()).hexdigest()[0:32]
 
-    email_filename = f"custom/custom_email_{email_template_hash}.source.html"
     email_id = f"zerver/emails/custom/custom_email_{email_template_hash}"
     markdown_email_base_template_path = "templates/zerver/emails/custom_email_base.pre.html"
-    html_source_template_path = f"templates/{email_id}.source.html"
+    html_template_path = f"templates/{email_id}.html"
     plain_text_template_path = f"templates/{email_id}.txt"
     subject_path = f"templates/{email_id}.subject.txt"
-    os.makedirs(os.path.dirname(html_source_template_path), exist_ok=True)
+    os.makedirs(os.path.dirname(html_template_path), exist_ok=True)
 
     # First, we render the Markdown input file just like our
     # user-facing docs with render_markdown_path.
@@ -536,18 +541,13 @@ def send_custom_email(
     rendered_input = render_markdown_path(plain_text_template_path.replace("templates/", ""))
 
     # And then extend it with our standard email headers.
-    with open(html_source_template_path, "w") as f:
+    with open(html_template_path, "w") as f:
         with open(markdown_email_base_template_path) as base_template:
-            # Note that we're doing a hacky non-Jinja2 substitution here;
-            # we do this because the normal render_markdown_path ordering
-            # doesn't commute properly with inline_email_css.
-            f.write(base_template.read().replace("{{ rendered_input }}", rendered_input))
+            f.write(base_template.read())
 
     with open(subject_path, "w") as f:
         f.write(get_header(options.get("subject"), parsed_email_template.get("subject"), "subject"))
 
-    inline_template(email_filename)
-
     # Finally, we send the actual emails.
     for user_profile in users:
         if options.get("admins_only") and not user_profile.is_realm_admin:
@@ -556,6 +556,7 @@ def send_custom_email(
             "realm_uri": user_profile.realm.uri,
             "realm_name": user_profile.realm.name,
             "unsubscribe_link": one_click_unsubscribe_link(user_profile, "marketing"),
+            "rendered_input": rendered_input,
         }
         with suppress(EmailNotDeliveredError):
             send_email(