thumbor: Add production installer/puppet support.

This commits adds the necessary puppet configuration and
installer/upgrade code for installing and managing the thumbor service
in production.  This configuration is gated by the 'thumbor.pp'
manifest being enabled (which is not yet the default), and so this
commit should have no effect in a default Zulip production environment
(or in the long term, in any Zulip production server that isn't using
thumbor).

Credit for this effort is shared by @TigorC (who initiated the work on
this project), @joshland (who did a great deal of work on this and got
it working during PyCon 2017) and @adnrs96, who completed the work.

puppet/zulip/files/nginx/zulip-include-app.d/thumbor.conf

@@ -0,0 +1,7 @@
+# This configuration proxies requests to /thumbor to a copy of the
+# thumbor service installed locally on the Zulip server.
+location /thumbor {
+    rewrite /thumbor/(.*) /$1 break;
+    proxy_pass http://thumbor;
+    include /etc/nginx/zulip-include/proxy;
+}

puppet/zulip/files/nginx/zulip-include-frontend/upstreams

@@ -14,3 +14,7 @@ upstream localhost_sso {
 upstream camo {
     server 127.0.0.1:9292;
 }
+
+upstream thumbor {
+    server localhost:9995;
+}

puppet/zulip/files/supervisor/conf.d/thumbor.conf

@@ -0,0 +1,12 @@
+[program:zulip-thumbor]
+command=/home/zulip/deployments/current/zulip-thumbor-venv/bin/thumbor --port=9995 --conf=/home/zulip/deployments/current/zthumbor/thumbor.conf
+user=zulip
+directory=/home/zulip/deployments/current/
+autostart=true
+autorestart=true
+startretries=3
+stopsignal=TERM
+redirect_stderr=true
+stdout_logfile=/var/log/zulip/thumbor.log
+stdout_logfile_maxbytes=20MB
+stdout_logfile_backups=3

puppet/zulip/manifests/thumbor.pp

@@ -0,0 +1,24 @@
+class zulip::thumbor {
+  include zulip::nginx
+  include zulip::supervisor
+
+  file { '/etc/supervisor/conf.d/thumbor.conf':
+    ensure  => file,
+    require => Package[supervisor],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/zulip/supervisor/conf.d/thumbor.conf',
+    notify  => Service['supervisor'],
+  }
+
+  file { '/etc/nginx/zulip-include/app.d/thumbor.conf':
+    ensure  => file,
+    require => Package['nginx-full'],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    notify  => Service['nginx'],
+    source  => 'puppet:///modules/zulip/nginx/zulip-include-app.d/thumbor.conf',
+  }
+}

scripts/lib/install

@@ -153,6 +153,7 @@ fi
 # Create and activate a virtualenv
 if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
     "$ZULIP_PATH"/scripts/lib/create-production-venv "$ZULIP_PATH"
+    "$ZULIP_PATH"/scripts/lib/create-thumbor-venv "$ZULIP_PATH"
 fi
 
 "$ZULIP_PATH"/scripts/lib/install-node

scripts/lib/upgrade-zulip-stage-2

@@ -60,6 +60,10 @@ if not os.path.exists((os.path.join(deploy_path, "zproject/prod_settings"))):
 subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-production-venv"),
                        deploy_path])
 
+# Setup the thumbor venv
+subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-thumbor-venv"),
+                       deploy_path])
+
 # Make sure the right version of node is installed
 subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node"),
                        deploy_path])
@@ -117,13 +121,19 @@ if not args.skip_migrations:
 if os.path.exists("/etc/supervisor/conf.d/zulip_db.conf"):
     subprocess.check_call(["supervisorctl", "stop", "process-fts-updates"], preexec_fn=su_to_zulip)
 
+core_server_services = ["zulip-django", "zulip-tornado", "zulip-senders:*"]
+worker_services = ["zulip-workers:*"]
+# Stop and start thumbor service only if thumbor is installed.
+if os.path.exists("/etc/supervisor/conf.d/thumbor.conf"):
+    core_server_services.append("zulip-thumbor")
+
 if not args.skip_puppet or migrations_needed:
     # By default, we shut down the service to apply migrations and
     # puppet changes, to minimize risk of issues due to inconsistent
     # state.
     logging.info("Stopping Zulip...")
-    subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*", "zulip-django",
-                           "zulip-tornado", "zulip-senders:*"], preexec_fn=su_to_zulip)
+    subprocess.check_call(["supervisorctl", "stop"] + core_server_services + worker_services,
+                          preexec_fn=su_to_zulip)
 
 if not args.skip_puppet:
     logging.info("Applying puppet changes...")

scripts/restart-server

@@ -25,11 +25,15 @@ subprocess.check_call(["./manage.py", "send_stats", "incr", "events.server_resta
 logging.info("Filling memcached caches")
 subprocess.check_call(["./manage.py", "fill_memcached_caches"])
 
+core_server_services = ["zulip-django", "zulip-tornado", "zulip-senders:*"]
+if os.path.exists("/etc/supervisor/conf.d/thumbor.conf"):
+    core_server_services.append("zulip-thumbor")
+
 # Restart the uWSGI and related processes via supervisorctl.
 logging.info("Stopping workers")
 subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*"])
 logging.info("Stopping server core")
-subprocess.check_call(["supervisorctl", "stop", "zulip-senders:* zulip-django zulip-tornado"])
+subprocess.check_call(["supervisorctl", "stop"] + core_server_services)
 
 current_symlink = os.path.join(DEPLOYMENTS_DIR, "current")
 last_symlink = os.path.join(DEPLOYMENTS_DIR, "last")
@@ -37,7 +41,7 @@ if os.readlink(current_symlink) != deploy_path:
     subprocess.check_call(["ln", '-nsf', os.readlink(current_symlink), last_symlink])
     subprocess.check_call(["ln", '-nsf', deploy_path, current_symlink])
 logging.info("Starting server core")
-subprocess.check_call(["supervisorctl", "start", "zulip-tornado zulip-django zulip-senders:*"])
+subprocess.check_call(["supervisorctl", "start"] + core_server_services)
 logging.info("Starting workers")
 subprocess.check_call(["supervisorctl", "start", "zulip-workers:*"])
 

zproject/prod_settings_template.py

@@ -482,6 +482,13 @@ CAMO_URI = '/external_content/'
 # Controls whether Zulip will rate-limit user requests.
 # RATE_LIMITING = True
 
+# By default, Zulip connects to the thumbor (the thumbnailing software
+# we use) service running locally on the machine.  If you're running
+# thumbor on a different server, you can configure that by setting
+# THUMBOR_HOST here.  Setting THUMBOR_HOST='' will disable
+# thumbnailing in Zulip.
+#THUMBOR_HOST = '127.0.0.1:9995'
+
 # Controls the Jitsi video call integration.  By default, the
 # integration uses the SaaS meet.jit.si server.  You can specify
 # your own Jitsi Meet server, or if you'd like to disable the