email-mirror-postfix: Handle 8-bit messages correctly.

Since JSON can’t represent bytes, we encode them with base64.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

scripts/lib/email-mirror-postfix

@@ -41,6 +41,7 @@ Also you can use optional keys to configure the script and change default values
 
 """
 import argparse
+import base64
 import json
 import os
 import posix
@@ -94,8 +95,8 @@ def send_email_mirror(
     if not rcpt_to:
         print("5.1.1 Bad destination mailbox address: No missed message email address.")
         exit(posix.EX_NOUSER)
-    msg_text = sys.stdin.read(MAX_ALLOWED_PAYLOAD + 1)
-    if len(msg_text) > MAX_ALLOWED_PAYLOAD:
+    msg_bytes = sys.stdin.buffer.read(MAX_ALLOWED_PAYLOAD + 1)
+    if len(msg_bytes) > MAX_ALLOWED_PAYLOAD:
         # We're not at EOF, reject large mail.
         print("5.3.4 Message too big for system: Max size is 25MiB")
         exit(posix.EX_DATAERR)
@@ -105,10 +106,6 @@ def send_email_mirror(
     if not shared_secret:
         shared_secret = secrets_file.get('secrets', 'shared_secret')
 
-    request_data = {
-        "recipient": rcpt_to,
-        "msg_text": msg_text,
-    }
     if test:
         exit(0)
 
@@ -122,8 +119,11 @@ def send_email_mirror(
         context = ssl.create_default_context()
         context.check_hostname = False
         context.verify_mode = ssl.CERT_NONE
-    data = {"data": json.dumps(request_data),
-            "secret": shared_secret}
+    data = {
+        "rcpt_to": rcpt_to,
+        "msg_base64": base64.b64encode(msg_bytes).decode(),
+        "secret": shared_secret,
+    }
     req = Request(url=urljoin(host, url), data=urlencode(data).encode('utf8'))
     try:
         urlopen(req, context=context)

zerver/lib/email_mirror.py

@@ -421,8 +421,7 @@ def validate_to_address(rcpt_to: str) -> None:
     else:
         decode_stream_email_address(rcpt_to)
 
-def mirror_email_message(data: Dict[str, str]) -> Dict[str, str]:
-    rcpt_to = data['recipient']
+def mirror_email_message(rcpt_to: str, msg_base64: str) -> Dict[str, str]:
     try:
         validate_to_address(rcpt_to)
     except ZulipEmailForwardError as e:
@@ -434,8 +433,8 @@ def mirror_email_message(data: Dict[str, str]) -> Dict[str, str]:
     queue_json_publish(
         "email_mirror",
         {
-            "message": data['msg_text'],
             "rcpt_to": rcpt_to,
+            "msg_base64": msg_base64,
         },
     )
     return {"status": "success"}

zerver/management/commands/send_to_email_mirror.py

@@ -1,8 +1,9 @@
+import base64
 import email
 import email.policy
 import os
 from email.message import EmailMessage
-from typing import Dict, Optional
+from typing import Optional
 
 import ujson
 from django.conf import settings
@@ -70,10 +71,10 @@ Example:
         message = self._parse_email_fixture(full_fixture_path)
         self._prepare_message(message, realm, stream)
 
-        data: Dict[str, str] = {}
-        data['recipient'] = message['To'].addresses[0].addr_spec
-        data['msg_text'] = message.as_string()
-        mirror_email_message(data)
+        mirror_email_message(
+            message['To'].addresses[0].addr_spec,
+            base64.b64encode(message.as_bytes()).decode(),
+        )
 
     def _does_fixture_path_exist(self, fixture_path: str) -> bool:
         return os.path.exists(fixture_path)

zerver/tests/test_email_mirror.py

@@ -1,3 +1,4 @@
+import base64
 import email.policy
 import os
 import subprocess
@@ -1148,26 +1149,24 @@ class TestEmailMirrorTornadoView(ZulipTestCase):
                              mock_queue_json_publish: mock.Mock) -> HttpResponse:
         mail_template = self.fixture_data('simple.txt', type='email')
         mail = mail_template.format(stream_to_address=to_address, sender=sender.delivery_email)
+        msg_base64 = base64.b64encode(mail.encode()).decode()
 
         def check_queue_json_publish(queue_name: str,
                                      event: Mapping[str, Any],
                                      processor: Optional[Callable[[Any], None]]=None) -> None:
             self.assertEqual(queue_name, "email_mirror")
-            self.assertEqual(event, {"rcpt_to": to_address, "message": mail})
+            self.assertEqual(event, {"rcpt_to": to_address, "msg_base64": msg_base64})
             MirrorWorker().consume(event)
 
             self.assertEqual(self.get_last_message().content,
                              "This is a plain-text message for testing Zulip.")
 
         mock_queue_json_publish.side_effect = check_queue_json_publish
-        request_data = {
-            "recipient": to_address,
-            "msg_text": mail,
+        post_data = {
+            "rcpt_to": to_address,
+            "msg_base64": msg_base64,
+            "secret": settings.SHARED_SECRET,
         }
-        post_data = dict(
-            data=ujson.dumps(request_data),
-            secret=settings.SHARED_SECRET,
-        )
         return self.client_post('/email_mirror_message', post_data)
 
     def test_success_stream(self) -> None:

zerver/tests/test_queue_worker.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import smtplib
 import time
@@ -307,7 +308,7 @@ class WorkerTest(ZulipTestCase):
         stream_to_address = encode_email_address(stream)
         data = [
             dict(
-                message='\xf3test',
+                msg_base64=base64.b64encode(b'\xf3test').decode(),
                 time=time.time(),
                 rcpt_to=stream_to_address,
             ),
@@ -334,7 +335,7 @@ class WorkerTest(ZulipTestCase):
         stream_to_address = encode_email_address(stream)
         data = [
             dict(
-                message='\xf3test',
+                msg_base64=base64.b64encode(b'\xf3test').decode(),
                 time=time.time(),
                 rcpt_to=stream_to_address,
             ),
@@ -361,7 +362,7 @@ class WorkerTest(ZulipTestCase):
                 with self.settings(EMAIL_GATEWAY_PATTERN="%s@example.com"):
                     address = 'mm' + ('x' * 32) + '@example.com'
                     event = dict(
-                        message='\xf3test',
+                        msg_base64=base64.b64encode(b'\xf3test').decode(),
                         time=time.time(),
                         rcpt_to=address,
                     )

zerver/views/email_mirror.py

@@ -1,22 +1,17 @@
-from typing import Dict
-
-import ujson
 from django.http import HttpRequest, HttpResponse
 
 from zerver.decorator import internal_notify_view
 from zerver.lib.email_mirror import mirror_email_message
 from zerver.lib.request import REQ, has_request_variables
 from zerver.lib.response import json_error, json_success
-from zerver.lib.validator import check_dict, check_string
 
 
 @internal_notify_view(False)
 @has_request_variables
-def email_mirror_message(request: HttpRequest,
-                         data: Dict[str, str]=REQ(validator=check_dict([
-                             ('recipient', check_string),
-                             ('msg_text', check_string)]))) -> HttpResponse:
-    result = mirror_email_message(ujson.loads(request.POST['data']))
+def email_mirror_message(
+    request: HttpRequest, rcpt_to: str = REQ(), msg_base64: str = REQ(),
+) -> HttpResponse:
+    result = mirror_email_message(rcpt_to, msg_base64)
     if result["status"] == "error":
         return json_error(result['msg'])
     return json_success()

zerver/worker/queue_processors.py

@@ -1,4 +1,5 @@
 # Documented in https://zulip.readthedocs.io/en/latest/subsystems/queuing.html
+import base64
 import copy
 import datetime
 import email
@@ -554,7 +555,10 @@ class DigestWorker(QueueProcessingWorker):  # nocoverage
 class MirrorWorker(QueueProcessingWorker):
     def consume(self, event: Mapping[str, Any]) -> None:
         rcpt_to = event['rcpt_to']
-        msg = email.message_from_string(event["message"], policy=email.policy.default)
+        msg = email.message_from_bytes(
+            base64.b64decode(event["msg_base64"]),
+            policy=email.policy.default,
+        )
         assert isinstance(msg, EmailMessage)  # https://github.com/python/typeshed/issues/2417
         if not is_missed_message_address(rcpt_to):
             # Missed message addresses are one-time use, so we don't need