process_queue: Recover gracefully after PostgreSQL restart.

- For threaded workers:
Django's autoreloader catches SIGQUIT(3) to reload the program. If
a process being watched by autoreloader exits with status code 3,
reloader will restart the process. To reload, we send SIGUSR1(10)
signal from consumers to a handler in process_queue which then
exits with status code 3.

- For single worker per process:
Catch the SIGUSR1 and quit; supervisorctl will restart the worker
automatically.

Fixes #5512

zerver/management/commands/process_queue.py

@@ -9,10 +9,12 @@ from django.core.management import CommandError
 from django.conf import settings
 from django.utils import autoreload
 from zerver.worker.queue_processors import get_worker, get_active_worker_queues
+import os
 import sys
 import signal
 import logging
 import threading
+import subprocess
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
@@ -35,6 +37,15 @@ class Command(BaseCommand):
         logging.basicConfig()
         logger = logging.getLogger('process_queue')
 
+        def exit_with_three(signal, frame):
+            # type: (int, FrameType) -> None
+            """
+            This process is watched by Django's autoreload, so exiting
+            with status code 3 will cause this process to restart.
+            """
+            logger.warn("SIGUSR1 received. Restarting this queue processor.")
+            sys.exit(3)
+
         if not settings.USING_RABBITMQ:
             # Make the warning silent when running the tests
             if settings.TEST_SUITE:
@@ -56,8 +67,10 @@ class Command(BaseCommand):
             logger.info('%d queue worker threads were launched' % (cnt,))
 
         if options['all']:
+            signal.signal(signal.SIGUSR1, exit_with_three)
             autoreload.main(run_threaded_workers, (get_active_worker_queues(), logger))
         elif options['multi_threaded']:
+            signal.signal(signal.SIGUSR1, exit_with_three)
             queues = options['multi_threaded']
             autoreload.main(run_threaded_workers, (queues, logger))
         else:
@@ -75,6 +88,7 @@ class Command(BaseCommand):
                 sys.exit(0)
             signal.signal(signal.SIGTERM, signal_handler)
             signal.signal(signal.SIGINT, signal_handler)
+            signal.signal(signal.SIGUSR1, signal_handler)
 
             worker.start()