[schema] Add ScheduledJob table, and update mandrill related code

ScheduledJobs with type Email displace the usual mandrill codepaths
in the Zulip Enterprise deploys

* Email-specific helper functions will appear in deliver_email.py
* 0058_auto__add_scheduledjob.py

(imported from commit 8db08d8a279600322acfdbed792dc1a676f7a0ab)

zerver/lib/actions.py

@@ -12,7 +12,8 @@ from zerver.models import Realm, RealmEmoji, Stream, UserProfile, UserActivity,
     to_dict_cache_key, get_realm, stringify_message_dict, bulk_get_recipients, \
     email_to_domain, email_to_username, display_recipient_cache_key, \
     get_stream_cache_key, to_dict_cache_key_id, is_super_user, \
-    UserActivityInterval, get_active_user_dicts_in_realm, RealmAlias
+    UserActivityInterval, get_active_user_dicts_in_realm, RealmAlias, \
+    ScheduledJob
 
 from django.db import transaction, IntegrityError
 from django.db.models import F, Q
@@ -2114,9 +2115,16 @@ def clear_followup_emails_queue(email, from_email=None, mail_client=None):
     `from_email` is a string representing the zulip email account used
     to send the email (for example `support@zulip.com` or `signups@zulip.com`)
     """
+    # Zulip Enterprise implementation
     if not mail_client:
+        items = ScheduledJob.objects.filter(type=ScheduledJob.EMAIL, filter_string__iexact = email)
+        if from_email is not None:
+            items = [item for item in items
+                     if ujson.loads(item.data).get('from_email') == from_email]
+        items.delete()
         return
 
+    # Mandrill implementation
     for email in mail_client.messages.list_scheduled(to=email):
         if from_email is not None and email.get('from_email') != from_email:
             continue
@@ -2140,9 +2148,25 @@ def send_future_email(recipients, email_html, email_text, subject,
     #            "tags": ["signup-reminders"],
     #            "to": [{'email':"acrefoot@zulip.com", 'name': "thingamajig"}]
     #            }
+
+    # Zulip Enterprise implementation
     if not mail_client:
+        if sender is None:
+            sender = {'email': "noreply@%" % (settings.EXTERNAL_HOST,), 'name': 'Zulip'}
+        for recipient in recipients:
+            email_fields = {'email_html': email_html,
+                            'email_subject': subject,
+                            'email_text': email_text,
+                            'recipient_email': recipient.get('email'),
+                            'recipient_name': recipient.get('name'),
+                            'sender_email': sender['email'],
+                            'sender_name': sender['name']}
+            ScheduledJob.objects.create(type=ScheduledJob.EMAIL, filter_string=recipient.get('email'),
+                                        data=ujson.dumps(email_fields),
+                                        scheduled_timestamp=datetime.datetime.utcnow() + delay)
         return
 
+    # Mandrill implementation
     if sender is None:
         sender = {'email': 'noreply@zulip.com', 'name': 'Zulip'}
 

zerver/lib/mandrill_client.py

@@ -4,7 +4,7 @@ from django.conf import settings
 MAIL_CLIENT = None
 
 def get_mandrill_client():
-    if settings.MANDRILL_API_KEY == '':
+    if settings.MANDRILL_API_KEY == '' or not settings.DEPLOYED or settings.ENTERPRISE:
         return None
 
     global MAIL_CLIENT

zerver/migrations/0058_auto__add_scheduledjob.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'ScheduledJob'
+        db.create_table(u'zerver_scheduledjob', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('scheduled_timestamp', self.gf('django.db.models.fields.DateTimeField')()),
+            ('type', self.gf('django.db.models.fields.PositiveSmallIntegerField')()),
+            ('data', self.gf('django.db.models.fields.TextField')()),
+            ('filter_id', self.gf('django.db.models.fields.IntegerField')(null=True)),
+            ('filter_string', self.gf('django.db.models.fields.CharField')(max_length=100)),
+        ))
+        db.send_create_signal(u'zerver', ['ScheduledJob'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'ScheduledJob'
+        db.delete_table(u'zerver_scheduledjob')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'zerver.appledevicetoken': {
+            'Meta': {'object_name': 'AppleDeviceToken'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_updated': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'auto_now': 'True', 'blank': 'True'}),
+            'token': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.client': {
+            'Meta': {'object_name': 'Client'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30', 'db_index': 'True'})
+        },
+        u'zerver.defaultstream': {
+            'Meta': {'unique_together': "(('realm', 'stream'),)", 'object_name': 'DefaultStream'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"}),
+            'stream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Stream']"})
+        },
+        u'zerver.huddle': {
+            'Meta': {'object_name': 'Huddle'},
+            'huddle_hash': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        u'zerver.message': {
+            'Meta': {'object_name': 'Message'},
+            'content': ('django.db.models.fields.TextField', [], {}),
+            'edit_history': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_edit_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'pub_date': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Recipient']"}),
+            'rendered_content': ('django.db.models.fields.TextField', [], {'null': 'True'}),
+            'rendered_content_version': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
+            'sender': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"}),
+            'sending_client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
+            'subject': ('django.db.models.fields.CharField', [], {'max_length': '60', 'db_index': 'True'})
+        },
+        u'zerver.mituser': {
+            'Meta': {'object_name': 'MitUser'},
+            'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'status': ('django.db.models.fields.IntegerField', [], {'default': '0'})
+        },
+        u'zerver.preregistrationuser': {
+            'Meta': {'object_name': 'PreregistrationUser'},
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'invited_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']", 'null': 'True'}),
+            'referred_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']", 'null': 'True'}),
+            'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'streams': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['zerver.Stream']", 'null': 'True', 'symmetrical': 'False'})
+        },
+        u'zerver.realm': {
+            'Meta': {'object_name': 'Realm'},
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '40', 'null': 'True'}),
+            'notifications_stream': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'+'", 'null': 'True', 'to': u"orm['zerver.Stream']"}),
+            'restricted_to_domain': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        u'zerver.realmalias': {
+            'Meta': {'object_name': 'RealmAlias'},
+            'domain': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']", 'null': 'True'})
+        },
+        u'zerver.realmemoji': {
+            'Meta': {'unique_together': "(('realm', 'name'),)", 'object_name': 'RealmEmoji'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'img_url': ('django.db.models.fields.TextField', [], {}),
+            'name': ('django.db.models.fields.TextField', [], {}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"})
+        },
+        u'zerver.recipient': {
+            'Meta': {'unique_together': "(('type', 'type_id'),)", 'object_name': 'Recipient'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'type': ('django.db.models.fields.PositiveSmallIntegerField', [], {'db_index': 'True'}),
+            'type_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'})
+        },
+        u'zerver.referral': {
+            'Meta': {'object_name': 'Referral'},
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.scheduledjob': {
+            'Meta': {'object_name': 'ScheduledJob'},
+            'data': ('django.db.models.fields.TextField', [], {}),
+            'filter_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
+            'filter_string': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'scheduled_timestamp': ('django.db.models.fields.DateTimeField', [], {}),
+            'type': ('django.db.models.fields.PositiveSmallIntegerField', [], {})
+        },
+        u'zerver.stream': {
+            'Meta': {'unique_together': "(('name', 'realm'),)", 'object_name': 'Stream'},
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email_token': ('django.db.models.fields.CharField', [], {'default': "'842cae53418b0f722c8f1769ffbc5982'", 'max_length': '32'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'invite_only': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '60', 'db_index': 'True'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"})
+        },
+        u'zerver.streamcolor': {
+            'Meta': {'object_name': 'StreamColor'},
+            'color': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'subscription': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Subscription']"})
+        },
+        u'zerver.subscription': {
+            'Meta': {'unique_together': "(('user_profile', 'recipient'),)", 'object_name': 'Subscription'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'color': ('django.db.models.fields.CharField', [], {'default': "'#c2c2c2'", 'max_length': '10'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'in_home_view': ('django.db.models.fields.NullBooleanField', [], {'default': 'True', 'null': 'True', 'blank': 'True'}),
+            'notifications': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'recipient': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Recipient']"}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.useractivity': {
+            'Meta': {'unique_together': "(('user_profile', 'client', 'query'),)", 'object_name': 'UserActivity'},
+            'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
+            'count': ('django.db.models.fields.IntegerField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_visit': ('django.db.models.fields.DateTimeField', [], {}),
+            'query': ('django.db.models.fields.CharField', [], {'max_length': '50', 'db_index': 'True'}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.useractivityinterval': {
+            'Meta': {'object_name': 'UserActivityInterval'},
+            'end': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'start': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.usermessage': {
+            'Meta': {'unique_together': "(('user_profile', 'message'),)", 'object_name': 'UserMessage'},
+            'flags': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Message']"}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.userpresence': {
+            'Meta': {'unique_together': "(('user_profile', 'client'),)", 'object_name': 'UserPresence'},
+            'client': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Client']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '1'}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {}),
+            'user_profile': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']"})
+        },
+        u'zerver.userprofile': {
+            'Meta': {'object_name': 'UserProfile'},
+            'alert_words': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'api_key': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'avatar_source': ('django.db.models.fields.CharField', [], {'default': "'G'", 'max_length': '1'}),
+            'bot_owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.UserProfile']", 'null': 'True', 'on_delete': 'models.SET_NULL'}),
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'db_index': 'True'}),
+            'enable_desktop_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enable_offline_email_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enable_offline_push_notifications': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enable_sounds': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'enter_sends': ('django.db.models.fields.NullBooleanField', [], {'default': 'False', 'null': 'True', 'blank': 'True'}),
+            'full_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'invites_granted': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'invites_used': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_bot': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_pointer_updater': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'last_reminder': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True'}),
+            'muted_topics': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'onboarding_steps': ('django.db.models.fields.TextField', [], {'default': "'[]'"}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'pointer': ('django.db.models.fields.IntegerField', [], {}),
+            'rate_limits': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '100'}),
+            'realm': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['zerver.Realm']"}),
+            'short_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'tutorial_status': ('django.db.models.fields.CharField', [], {'default': "'W'", 'max_length': '1'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"})
+        }
+    }
+
+    complete_apps = ['zerver']

zerver/models.py

@@ -998,3 +998,19 @@ class Referral(models.Model):
     user_profile = models.ForeignKey(UserProfile)
     email = models.EmailField(blank=False, null=False)
     timestamp = models.DateTimeField(auto_now_add=True, null=False)
+
+# This table only gets used on Zulip Enterprise instances
+# For reasons of deliverability (and sending from multiple email addresses),
+# we will still send from mandrill when we send things from the (staging.)zulip.com install
+class ScheduledJob(models.Model):
+    scheduled_timestamp = models.DateTimeField(auto_now_add=False, null=False)
+    type = models.PositiveSmallIntegerField()
+    # Valid types are {email}
+    # for EMAIL, filter_string is recipient_email
+    EMAIL = 1
+
+    # JSON representation of the job's data. Be careful, as we are not relying on Django to do validation
+    data = models.TextField()
+    # Kind if like a ForeignKey, but table is determined by type.
+    filter_id = models.IntegerField(null=True)
+    filter_string = models.CharField(max_length=100)

zproject/local_settings.py

@@ -56,6 +56,7 @@ EMAIL_HOST_USER = 'zulip@zulip.com'
 EMAIL_HOST_PASSWORD = 'xxxxxxxxxxxxxxxx'
 EMAIL_PORT = 587
 EMAIL_USE_TLS = True
+ALLOW_ARBITRARY_SENDERS_LOCAL_EMAIL = False
 
 SESSION_SERIALIZER = "django.contrib.sessions.serializers.PickleSerializer"
 

zproject/settings.py

@@ -681,6 +681,10 @@ TEMPLATE_CONTEXT_PROCESSORS = (
 ACCOUNT_ACTIVATION_DAYS=7
 
 DEFAULT_FROM_EMAIL = "Zulip <zulip@zulip.com>"
+# In some email setups, a single inbox may be setup to be a wildcard, so in that case allow
+# differing senders. Otherwise, *always* use the DEFAULT_FROM_EMAIL when not sending via
+# mandrill (i.e. in the localserver case)
+ALLOW_ARBITRARY_SENDERS_LOCAL_EMAIL = False
 
 LOGIN_REDIRECT_URL='/'
 OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'