Refactor email-mirror to handle running on any machine

(imported from commit 2971449ceaacb564770e66874fc095f77e68d445)

zerver/lib/actions.py

@@ -1547,12 +1547,21 @@ def encode_email_address_helper(name, email_token):
     # ordinal of that character, padded with zeroes to the maximum number of
     # bytes of a UTF-8 encoded Unicode character.
     encoded_name = re.sub("\W", lambda x: "%" + str(ord(x.group(0))).zfill(4), name)
-    return "%s+%s@streams.zulip.com" % (encoded_name, email_token)
+    encoded_token = "%s+%s" % (encoded_name, email_token)
+    return settings.EMAIL_GATEWAY_PATTERN % (encoded_token,)
 
 def decode_email_address(email):
-    # Perform the reverse of encode_email_address. Only the stream name will be
-    # transformed.
-    return re.sub("%\d{4}", lambda x: unichr(int(x.group(0)[1:])), email)
+    # Perform the reverse of encode_email_address. Returns a tuple of (streamname, email_token)
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split('%s')]
+    match_email_re = re.compile("(.*?)".join(pattern_parts))
+    match = match_email_re.match(email)
+
+    if not match:
+        return None
+
+    token = match.group(1)
+    decoded_token = re.sub("%\d{4}", lambda x: unichr(int(x.group(0)[1:])), token)
+    return decoded_token.split('+')
 
 # In general, it's better to avoid using .values() because it makes
 # the code pretty ugly, but in this case, it has significant

zerver/management/commands/email-mirror.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 """
-Forward messages sent to @streams.zulip.com to Zulip.
+Forward messages sent to the configured email gateway to Zulip.
 
 Messages to that address go to the Inbox of emailgateway@zulip.com.
 
@@ -37,13 +37,6 @@ import html2text
 sys.path.insert(0, path.join(path.dirname(__file__), "../../../api"))
 import zulip
 
-GATEWAY_EMAIL = "emailgateway@zulip.com"
-# Application-specific password.
-PASSWORD = "xxxxxxxxxxxxxxxx"
-
-SERVER = "imap.gmail.com"
-PORT = 993
-
 ## Setup ##
 
 log_format = "%(asctime)s: %(message)s"
@@ -57,28 +50,36 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 logger.addHandler(file_handler)
 
-email_gateway_user = get_user_profile_by_email(GATEWAY_EMAIL)
+email_gateway_user = get_user_profile_by_email(settings.EMAIL_GATEWAY_BOT_ZULIP_USER)
 api_key = email_gateway_user.api_key
+
 if settings.DEPLOYED:
-    staging_client = zulip.Client(
-        site="https://staging.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
-    prod_client = zulip.Client(
-        site="https://api.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
-    inbox = "INBOX"
+    staging_api_client = zulip.Client(
+            site="https://staging.zulip.com",
+            email=settings.EMAIL_GATEWAY_BOT_ZULIP_USER,
+            api_key=api_key)
+
+
+    api_client = zulip.Client(
+            site=settings.EXTERNAL_HOST,
+            email=settings.EMAIL_GATEWAY_BOT_ZULIP_USER,
+            api_key=api_key)
 else:
-    staging_client = prod_client = zulip.Client(
-        site="http://localhost:9991/api", email=GATEWAY_EMAIL, api_key=api_key)
-    inbox = "Test"
+    api_client = staging_api_client = zulip.Client(
+            site=settings.EXTERNAL_HOST,
+            email=settings.EMAIL_GATEWAY_BOT_ZULIP_USER,
+            api_key=api_key)
 
 def redact_stream(error_message):
-    stream_match = re.search("(\S+?)\+(\S+?)@streams.zulip.com", error_message)
+    domain = settings.EMAIL_GATEWAY_PATTERN.rsplit('@')[-1]
+    stream_match = re.search(r'\b(.*?)@' + domain, error_message)
     if stream_match:
         stream_name = stream_match.groups()[0]
         return error_message.replace(stream_name, "X" * len(stream_name))
     return error_message
 
 def report_to_zulip(error_message):
-    error_stream = Stream.objects.get(name="errors", realm__domain="zulip.com")
+    error_stream = Stream.objects.get(name="errors", realm__domain=settings.ADMIN_DOMAIN)
     send_zulip(error_stream, "email mirror error",
                """~~~\n%s\n~~~""" % (error_message,))
 
@@ -103,13 +104,13 @@ class ZulipEmailForwardError(Exception):
     pass
 
 def send_zulip(stream, topic, content):
-    if stream.realm.domain in ["zulip.com", "wdaher.com"]:
-        api_client = staging_client
-    else:
-        api_client = prod_client
-
     # TODO: restrictions on who can send? Consider: cross-realm
     # messages, private streams.
+    if stream.realm.domain == 'zulip.com':
+        client = staging_api_client
+    else:
+        client = api_client
+
     message_data = {
         "type": "stream",
         # TODO: handle rich formatting.
@@ -119,7 +120,7 @@ def send_zulip(stream, topic, content):
         "domain": stream.realm.domain
         }
 
-    response = api_client.send_message(message_data)
+    response = client.send_message(message_data)
     if response["result"] != "success":
         raise ZulipEmailForwardError(response["msg"])
 
@@ -189,12 +190,9 @@ def extract_and_upload_attachments(message):
     return "\n".join(attachment_links)
 
 def extract_and_validate(email):
-    # Recipient is of the form
-    # <stream name>+<regenerable stream token>@streams.zulip.com
     try:
-        stream_name_and_token = decode_email_address(email).rsplit("@", 1)[0]
-        stream_name, token = stream_name_and_token.rsplit("+", 1)
-    except ValueError:
+        stream_name, token = decode_email_address(email)
+    except TypeError:
         raise ZulipEmailForwardError("Malformed email recipient " + email)
 
     if not valid_stream(stream_name, token):
@@ -214,13 +212,24 @@ def delete(result, proto):
     return proto.close().addCallback(logout, proto)
 
 def find_emailgateway_recipient(message):
-    # We can't use Delivered-To; that is emailgateway@zulip.com.
-    recipients = message.get_all("X-Gm-Original-To", [])
+    # We can't use Delivered-To; if there is a X-Gm-Original-To
+    # it is more accurate, so try to find the most-accurate
+    # recipient list in descending priority order
+    recipient_headers = ["X-Gm-Original-To", "Delivered-To", "To"]
+    recipients = []
+    for recipient_header in recipient_headers:
+        r = message.get_all(recipient_header, None)
+        if r:
+            recipients = r
+            break
+
+    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split('%s')]
+    match_email_re = re.compile(".*?".join(pattern_parts))
     for recipient_email in recipients:
-        if recipient_email.lower().endswith("@streams.zulip.com"):
+        if match_email_re.match(recipient_email):
             return recipient_email
 
-    raise ZulipEmailForwardError("Missing recipient @streams.zulip.com")
+    raise ZulipEmailForwardError("Missing recipient in mirror email")
 
 def fetch(result, proto, mailboxes):
     if not result:
@@ -264,7 +273,7 @@ def examine_mailbox(result, proto, mailbox):
 
 def select_mailbox(result, proto):
     # Select which mailbox we care about.
-    mbox = filter(lambda x: inbox in x[2], result)[0][2]
+    mbox = filter(lambda x: settings.EMAIL_GATEWAY_IMAP_FOLDER in x[2], result)[0][2]
     return proto.select(mbox).addCallback(examine_mailbox, proto, result)
 
 def list_mailboxes(res, proto):
@@ -272,7 +281,7 @@ def list_mailboxes(res, proto):
     return proto.list("","*").addCallback(select_mailbox, proto)
 
 def connected(proto):
-    d = proto.login(GATEWAY_EMAIL, PASSWORD)
+    d = proto.login(settings.EMAIL_GATEWAY_LOGIN, settings.EMAIL_GATEWAY_PASSWORD)
     d.addCallback(list_mailboxes, proto)
     d.addErrback(login_failed)
     return d
@@ -285,12 +294,12 @@ def done(_):
 
 def main():
     imap_client = protocol.ClientCreator(reactor, imap4.IMAP4Client)
-    d = imap_client.connectSSL(SERVER, PORT, ssl.ClientContextFactory())
+    d = imap_client.connectSSL(settings.EMAIL_GATEWAY_IMAP_SERVER, settings.EMAIL_GATEWAY_IMAP_PORT, ssl.ClientContextFactory())
     d.addCallbacks(connected, login_failed)
     d.addBoth(done)
 
 class Command(BaseCommand):
-    help = """Forward emails set to @streams.zulip.com to Zulip.
+    help = """Forward emails sent to the configured email gateway to Zulip.
 
 Run this command out of a cron job.
 """

zerver/views/__init__.py

@@ -1368,7 +1368,7 @@ def send_message_backend(request, user_profile,
         return json_error("User not authorized for this query")
 
     realm = None
-    if domain:
+    if domain and domain != user_profile.realm.domain:
         if not is_super_user:
             # The email gateway bot needs to be able to send messages in
             # any realm.

zproject/local_settings.py

@@ -117,3 +117,22 @@ else:
     APNS_SANDBOX = "push_sandbox"
     APNS_FEEDBACK = "feedback_sandbox"
     APNS_CERT_FILE = "/etc/ssl/django-private/apns-dev.pem"
+
+# Administrator domain for this install
+ADMIN_DOMAIN = "zulip.com"
+
+# Email mirror configuration
+# The email of the Zulip bot that the email gateway
+# should post as
+EMAIL_GATEWAY_BOT_ZULIP_USER = "emailgateway-bot@zulip.com"
+
+EMAIL_GATEWAY_LOGIN = "emailgateway@zulip.com"
+EMAIL_GATEWAY_PASSWORD = "xxxxxxxxxxxxxxxx"
+EMAIL_GATEWAY_IMAP_SERVER = "imap.gmail.com"
+EMAIL_GATEWAY_IMAP_PORT = 993
+EMAIL_GATEWAY_IMAP_FOLDER = "INBOX"
+
+# The email address pattern to use for auto-generated stream emails
+# The %s will be replaced with a unique token, and the resulting email
+# must be delivered to the Inbox of the EMAIL_GATEWAY_LOGIN account above
+EMAIL_GATEWAY_PATTERN = "%s@streams.zulip.com"