scheduledmessages: Handle multiple servers running this delivery job.

We add conditional infinite sleep to this delivery job as a means to
handle case of multiple servers in service to a realm running this
job. In such a scenerio race conditions might arise leading to
multiple deliveries for same message. This way we try to match the
behaviour of what other jobs do in such a case.
Note: We should eventually do something to make such jobs work
while being running on multiple servers.

zerver/management/commands/deliver_scheduled_messages.py

@@ -22,6 +22,11 @@ class Command(BaseCommand):
     help = """Deliver scheduled messages from the ScheduledMessage table.
 Run this command under supervisor.
 
+This management command is run via supervisor.  Do not run on multiple
+machines, as you may encounter multiple sends in a specific race
+condition.  (Alternatively, you can set `EMAIL_DELIVERER_DISABLED=True`
+on all but one machine to make the command have no effect.)
+
 Usage: ./manage.py deliver_scheduled_messages
 """
 
@@ -46,6 +51,15 @@ Usage: ./manage.py deliver_scheduled_messages
                 'realm': scheduled_message.realm}
 
     def handle(self, *args: Any, **options: Any) -> None:
+
+        if settings.EMAIL_DELIVERER_DISABLED:
+            # Here doing a check and sleeping indefinitely on this setting might
+            # not sound right. Actually we do this check to avoid running this
+            # process on every server that might be in service to a realm. See
+            # the comment in zproject/settings.py file about renaming this setting.
+            while True:
+                time.sleep(10*9)
+
         with lockfile("/tmp/zulip_scheduled_message_deliverer.lockfile"):
             while True:
                 messages_to_deliver = ScheduledMessage.objects.filter(

zproject/settings.py

@@ -341,11 +341,17 @@ DEFAULT_SETTINGS.update({
     'INVITES_NEW_REALM_DAYS': 7,
 
     # Controls for which links are published in portico footers/headers/etc.
-    'EMAIL_DELIVERER_DISABLED': False,
     'REGISTER_LINK_DISABLED': None,
     'LOGIN_LINK_DISABLED': False,
     'FIND_TEAM_LINK_DISABLED': True,
 
+    # Controls if the server should run certain jobs like deliver_email or
+    # deliver_scheduled_messages. This setting in long term is meant for
+    # handling jobs for which we don't have a means of establishing a locking
+    # mechanism that works with multiple servers running these jobs.
+    # TODO: We should rename this setting so that it reflects its purpose actively.
+    'EMAIL_DELIVERER_DISABLED': False,
+
     # What domains to treat like the root domain
     'ROOT_SUBDOMAIN_ALIASES': ["www"],
     # Whether the root domain is a landing page or can host a realm.