Run queue processors multithreaded in production if system memory <3.5GB.

While running queue processors multithreaded will limit the
performance available to very small systems, it's easy to fix that by
adding more RAM, and previously, Zulip didn't work on such systems at
all, so this is unambiguously an improvement there.

Fixes #32.
Fixes #34.

(Commit message expanded significantly by tabbott.)

zerver/management/commands/process_queue.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
 from types import FrameType
-from typing import Any
+from typing import Any, List
 
 from argparse import ArgumentParser
 from django.core.management.base import BaseCommand
@@ -23,6 +23,10 @@ class Command(BaseCommand):
                             help="worker label")
         parser.add_argument('--all', dest="all", action="store_true", default=False,
                             help="run all queues")
+        parser.add_argument('--multi_threaded', nargs='+',
+                            metavar='<list of queue name>',
+                            type=str, required=False,
+                            help="list of queue to process")
 
     help = "Runs a queue processing worker"
 
@@ -39,19 +43,23 @@ class Command(BaseCommand):
                 logger.error("Cannot run a queue processor when USING_RABBITMQ is False!")
             sys.exit(1)
 
-        def run_threaded_workers(logger):
-            # type: (logging.Logger) -> None
+        def run_threaded_workers(queues, logger):
+            # type: (List[str], logging.Logger) -> None
             cnt = 0
-            for queue_name in get_active_worker_queues():
+            for queue_name in queues:
                 if not settings.DEVELOPMENT:
                     logger.info('launching queue worker thread ' + queue_name)
                 cnt += 1
                 td = Threaded_worker(queue_name)
                 td.start()
+            assert len(queues) == cnt
             logger.info('%d queue worker threads were launched' % (cnt,))
 
         if options['all']:
-            autoreload.main(run_threaded_workers, (logger,))
+            autoreload.main(run_threaded_workers, (get_active_worker_queues(), logger))
+        elif options['multi_threaded']:
+            queues = options['multi_threaded']
+            autoreload.main(run_threaded_workers, (queues, logger))
         else:
             queue_name = options['queue_name']
             worker_num = options['worker_num']