From b1a57d144f3d13236a15e24fdb6a475a54f76a29 Mon Sep 17 00:00:00 2001 From: Joshua Schmidlkofer Date: Tue, 23 May 2017 17:46:52 -0700 Subject: [PATCH] thumbor: Add production installer/puppet support. This commits adds the necessary puppet configuration and installer/upgrade code for installing and managing the thumbor service in production. This configuration is gated by the 'thumbor.pp' manifest being enabled (which is not yet the default), and so this commit should have no effect in a default Zulip production environment (or in the long term, in any Zulip production server that isn't using thumbor). Credit for this effort is shared by @TigorC (who initiated the work on this project), @joshland (who did a great deal of work on this and got it working during PyCon 2017) and @adnrs96, who completed the work. --- .../nginx/zulip-include-app.d/thumbor.conf | 7 ++++++ .../nginx/zulip-include-frontend/upstreams | 4 ++++ .../files/supervisor/conf.d/thumbor.conf | 12 ++++++++++ puppet/zulip/manifests/thumbor.pp | 24 +++++++++++++++++++ scripts/lib/install | 1 + scripts/lib/upgrade-zulip-stage-2 | 14 +++++++++-- scripts/restart-server | 8 +++++-- zproject/prod_settings_template.py | 7 ++++++ 8 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 puppet/zulip/files/nginx/zulip-include-app.d/thumbor.conf create mode 100644 puppet/zulip/files/supervisor/conf.d/thumbor.conf create mode 100644 puppet/zulip/manifests/thumbor.pp diff --git a/puppet/zulip/files/nginx/zulip-include-app.d/thumbor.conf b/puppet/zulip/files/nginx/zulip-include-app.d/thumbor.conf new file mode 100644 index 0000000000..c80ad4f078 --- /dev/null +++ b/puppet/zulip/files/nginx/zulip-include-app.d/thumbor.conf @@ -0,0 +1,7 @@ +# This configuration proxies requests to /thumbor to a copy of the +# thumbor service installed locally on the Zulip server. +location /thumbor { + rewrite /thumbor/(.*) /$1 break; + proxy_pass http://thumbor; + include /etc/nginx/zulip-include/proxy; +} diff --git a/puppet/zulip/files/nginx/zulip-include-frontend/upstreams b/puppet/zulip/files/nginx/zulip-include-frontend/upstreams index f17716180d..db72f51b3f 100644 --- a/puppet/zulip/files/nginx/zulip-include-frontend/upstreams +++ b/puppet/zulip/files/nginx/zulip-include-frontend/upstreams @@ -14,3 +14,7 @@ upstream localhost_sso { upstream camo { server 127.0.0.1:9292; } + +upstream thumbor { + server localhost:9995; +} diff --git a/puppet/zulip/files/supervisor/conf.d/thumbor.conf b/puppet/zulip/files/supervisor/conf.d/thumbor.conf new file mode 100644 index 0000000000..637d7e17dc --- /dev/null +++ b/puppet/zulip/files/supervisor/conf.d/thumbor.conf @@ -0,0 +1,12 @@ +[program:zulip-thumbor] +command=/home/zulip/deployments/current/zulip-thumbor-venv/bin/thumbor --port=9995 --conf=/home/zulip/deployments/current/zthumbor/thumbor.conf +user=zulip +directory=/home/zulip/deployments/current/ +autostart=true +autorestart=true +startretries=3 +stopsignal=TERM +redirect_stderr=true +stdout_logfile=/var/log/zulip/thumbor.log +stdout_logfile_maxbytes=20MB +stdout_logfile_backups=3 diff --git a/puppet/zulip/manifests/thumbor.pp b/puppet/zulip/manifests/thumbor.pp new file mode 100644 index 0000000000..3c7e6a85e3 --- /dev/null +++ b/puppet/zulip/manifests/thumbor.pp @@ -0,0 +1,24 @@ +class zulip::thumbor { + include zulip::nginx + include zulip::supervisor + + file { '/etc/supervisor/conf.d/thumbor.conf': + ensure => file, + require => Package[supervisor], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/supervisor/conf.d/thumbor.conf', + notify => Service['supervisor'], + } + + file { '/etc/nginx/zulip-include/app.d/thumbor.conf': + ensure => file, + require => Package['nginx-full'], + owner => 'root', + group => 'root', + mode => '0644', + notify => Service['nginx'], + source => 'puppet:///modules/zulip/nginx/zulip-include-app.d/thumbor.conf', + } +} diff --git a/scripts/lib/install b/scripts/lib/install index b5de1bea4f..d4b51739d3 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -153,6 +153,7 @@ fi # Create and activate a virtualenv if [ "$VIRTUALENV_NEEDED" = "yes" ]; then "$ZULIP_PATH"/scripts/lib/create-production-venv "$ZULIP_PATH" + "$ZULIP_PATH"/scripts/lib/create-thumbor-venv "$ZULIP_PATH" fi "$ZULIP_PATH"/scripts/lib/install-node diff --git a/scripts/lib/upgrade-zulip-stage-2 b/scripts/lib/upgrade-zulip-stage-2 index b39e771f6a..16a7d1bb6d 100755 --- a/scripts/lib/upgrade-zulip-stage-2 +++ b/scripts/lib/upgrade-zulip-stage-2 @@ -60,6 +60,10 @@ if not os.path.exists((os.path.join(deploy_path, "zproject/prod_settings"))): subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-production-venv"), deploy_path]) +# Setup the thumbor venv +subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-thumbor-venv"), + deploy_path]) + # Make sure the right version of node is installed subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node"), deploy_path]) @@ -117,13 +121,19 @@ if not args.skip_migrations: if os.path.exists("/etc/supervisor/conf.d/zulip_db.conf"): subprocess.check_call(["supervisorctl", "stop", "process-fts-updates"], preexec_fn=su_to_zulip) +core_server_services = ["zulip-django", "zulip-tornado", "zulip-senders:*"] +worker_services = ["zulip-workers:*"] +# Stop and start thumbor service only if thumbor is installed. +if os.path.exists("/etc/supervisor/conf.d/thumbor.conf"): + core_server_services.append("zulip-thumbor") + if not args.skip_puppet or migrations_needed: # By default, we shut down the service to apply migrations and # puppet changes, to minimize risk of issues due to inconsistent # state. logging.info("Stopping Zulip...") - subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*", "zulip-django", - "zulip-tornado", "zulip-senders:*"], preexec_fn=su_to_zulip) + subprocess.check_call(["supervisorctl", "stop"] + core_server_services + worker_services, + preexec_fn=su_to_zulip) if not args.skip_puppet: logging.info("Applying puppet changes...") diff --git a/scripts/restart-server b/scripts/restart-server index e0f86283e4..4a6bdfafdf 100755 --- a/scripts/restart-server +++ b/scripts/restart-server @@ -25,11 +25,15 @@ subprocess.check_call(["./manage.py", "send_stats", "incr", "events.server_resta logging.info("Filling memcached caches") subprocess.check_call(["./manage.py", "fill_memcached_caches"]) +core_server_services = ["zulip-django", "zulip-tornado", "zulip-senders:*"] +if os.path.exists("/etc/supervisor/conf.d/thumbor.conf"): + core_server_services.append("zulip-thumbor") + # Restart the uWSGI and related processes via supervisorctl. logging.info("Stopping workers") subprocess.check_call(["supervisorctl", "stop", "zulip-workers:*"]) logging.info("Stopping server core") -subprocess.check_call(["supervisorctl", "stop", "zulip-senders:* zulip-django zulip-tornado"]) +subprocess.check_call(["supervisorctl", "stop"] + core_server_services) current_symlink = os.path.join(DEPLOYMENTS_DIR, "current") last_symlink = os.path.join(DEPLOYMENTS_DIR, "last") @@ -37,7 +41,7 @@ if os.readlink(current_symlink) != deploy_path: subprocess.check_call(["ln", '-nsf', os.readlink(current_symlink), last_symlink]) subprocess.check_call(["ln", '-nsf', deploy_path, current_symlink]) logging.info("Starting server core") -subprocess.check_call(["supervisorctl", "start", "zulip-tornado zulip-django zulip-senders:*"]) +subprocess.check_call(["supervisorctl", "start"] + core_server_services) logging.info("Starting workers") subprocess.check_call(["supervisorctl", "start", "zulip-workers:*"]) diff --git a/zproject/prod_settings_template.py b/zproject/prod_settings_template.py index b550da8fbc..7095096baa 100644 --- a/zproject/prod_settings_template.py +++ b/zproject/prod_settings_template.py @@ -482,6 +482,13 @@ CAMO_URI = '/external_content/' # Controls whether Zulip will rate-limit user requests. # RATE_LIMITING = True +# By default, Zulip connects to the thumbor (the thumbnailing software +# we use) service running locally on the machine. If you're running +# thumbor on a different server, you can configure that by setting +# THUMBOR_HOST here. Setting THUMBOR_HOST='' will disable +# thumbnailing in Zulip. +#THUMBOR_HOST = '127.0.0.1:9995' + # Controls the Jitsi video call integration. By default, the # integration uses the SaaS meet.jit.si server. You can specify # your own Jitsi Meet server, or if you'd like to disable the