From 7666ff603d5dae0deb4847684c37d8e454563a9b Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Wed, 31 Aug 2022 15:46:08 -0700 Subject: [PATCH] sharding: Configure Tornado sharding with nginx map. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://nginx.org/en/docs/http/ngx_http_map_module.html Since Puppet doesn’t manage the contents of nginx_sharding.conf after its initial creation, it needs to be renamed so we can give it different default contents. Signed-off-by: Anders Kaseorg --- puppet/zulip/manifests/tornado_sharding.pp | 13 +++++++--- .../nginx/zulip-enterprise.template.erb | 2 +- .../files/nginx/sites-available/zulip | 2 +- .../files/nginx/sites-available/zulip-staging | 2 +- scripts/lib/sharding.py | 26 +++++++++---------- scripts/refresh-sharding-and-restart | 8 +++--- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/puppet/zulip/manifests/tornado_sharding.pp b/puppet/zulip/manifests/tornado_sharding.pp index cf88841cb9..5da3c310af 100644 --- a/puppet/zulip/manifests/tornado_sharding.pp +++ b/puppet/zulip/manifests/tornado_sharding.pp @@ -5,15 +5,22 @@ class zulip::tornado_sharding { # with the correct default content for the "only one shard" setup. For this # reason they use "replace => false", because the files are managed by # the sharding script afterwards and Puppet shouldn't overwrite them. - file { '/etc/zulip/nginx_sharding.conf': + file { '/etc/zulip/nginx_sharding_map.conf': ensure => file, owner => 'root', group => 'root', mode => '0644', notify => Service['nginx'], - content => "set \$tornado_server http://tornado;\n", + content => @(EOT), + map "" $tornado_server { + default http://tornado; + } + | EOT replace => false, } + file { '/etc/zulip/nginx_sharding.conf': + ensure => absent, + } file { '/etc/zulip/sharding.json': ensure => file, require => User['zulip'], @@ -29,7 +36,7 @@ class zulip::tornado_sharding { exec { 'stage_updated_sharding': command => "${::zulip_scripts_path}/lib/sharding.py", onlyif => "${::zulip_scripts_path}/lib/sharding.py --errors-ok", - require => [File['/etc/zulip/nginx_sharding.conf'], File['/etc/zulip/sharding.json']], + require => [File['/etc/zulip/nginx_sharding_map.conf'], File['/etc/zulip/sharding.json']], logoutput => true, loglevel => warning, } diff --git a/puppet/zulip/templates/nginx/zulip-enterprise.template.erb b/puppet/zulip/templates/nginx/zulip-enterprise.template.erb index 7276c7ac27..4f376d9f19 100644 --- a/puppet/zulip/templates/nginx/zulip-enterprise.template.erb +++ b/puppet/zulip/templates/nginx/zulip-enterprise.template.erb @@ -13,6 +13,7 @@ server { <% end -%> include /etc/nginx/zulip-include/upstreams; +include /etc/zulip/nginx_sharding_map.conf; server { <% if @nginx_http_only -%> @@ -30,7 +31,6 @@ server { alias /home/zulip/local-static; } - include /etc/zulip/nginx_sharding.conf; include /etc/nginx/zulip-include/certbot; include /etc/nginx/zulip-include/app; } diff --git a/puppet/zulip_ops/files/nginx/sites-available/zulip b/puppet/zulip_ops/files/nginx/sites-available/zulip index 5de9a13d37..70fe13e8ae 100644 --- a/puppet/zulip_ops/files/nginx/sites-available/zulip +++ b/puppet/zulip_ops/files/nginx/sites-available/zulip @@ -1,4 +1,5 @@ include /etc/nginx/zulip-include/upstreams; +include /etc/zulip/nginx_sharding_map.conf; server { listen 443 ssl http2; @@ -15,6 +16,5 @@ server { server_name zulipchat.com *.zulipchat.com; - include /etc/zulip/nginx_sharding.conf; include /etc/nginx/zulip-include/app; } diff --git a/puppet/zulip_ops/files/nginx/sites-available/zulip-staging b/puppet/zulip_ops/files/nginx/sites-available/zulip-staging index c7a8e8cd64..459a831329 100644 --- a/puppet/zulip_ops/files/nginx/sites-available/zulip-staging +++ b/puppet/zulip_ops/files/nginx/sites-available/zulip-staging @@ -6,6 +6,7 @@ server { } include /etc/nginx/zulip-include/upstreams; +include /etc/zulip/nginx_sharding_map.conf; server { listen 443 ssl http2; @@ -22,6 +23,5 @@ server { server_name zulipstaging.com; - include /etc/zulip/nginx_sharding.conf; include /etc/nginx/zulip-include/app; } diff --git a/scripts/lib/sharding.py b/scripts/lib/sharding.py index 127a9aa9d3..48cb61c07b 100755 --- a/scripts/lib/sharding.py +++ b/scripts/lib/sharding.py @@ -5,7 +5,7 @@ import json import os import subprocess import sys -from typing import Any, Dict +from typing import Dict BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(BASE_DIR) @@ -16,17 +16,13 @@ setup_path() from scripts.lib.zulip_tools import get_config_file, get_tornado_ports -def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None: - f.write( - f"""if ($host = '{host}') {{ - set $tornado_server http://tornado{port}; -}}\n""" - ) +def nginx_quote(s: str) -> str: + return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"' # Basic system to do Tornado sharding. Writes two output .tmp files that need # to be renamed to the following files to finalize the changes: -# * /etc/zulip/nginx_sharding.conf; nginx needs to be reloaded after changing. +# * /etc/zulip/nginx_sharding_map.conf; nginx needs to be reloaded after changing. # * /etc/zulip/sharding.json; supervisor Django process needs to be reloaded # after changing. TODO: We can probably make this live-reload by statting the file. # @@ -40,16 +36,19 @@ def write_updated_configs() -> None: sorted(ports) == expected_ports ), f"ports ({sorted(ports)}) must be contiguous, starting with 9800" - with open("/etc/zulip/nginx_sharding.conf.tmp", "w") as nginx_sharding_conf_f, open( + with open("/etc/zulip/nginx_sharding_map.conf.tmp", "w") as nginx_sharding_conf_f, open( "/etc/zulip/sharding.json.tmp", "w" ) as sharding_json_f: if len(ports) == 1: - nginx_sharding_conf_f.write("set $tornado_server http://tornado;\n") + nginx_sharding_conf_f.write('map "" $tornado_server {\n') + nginx_sharding_conf_f.write(" default http://tornado;\n") + nginx_sharding_conf_f.write("}\n") sharding_json_f.write("{}\n") return - nginx_sharding_conf_f.write("set $tornado_server http://tornado9800;\n") + nginx_sharding_conf_f.write("map $http_host $tornado_server {\n") + nginx_sharding_conf_f.write(" default http://tornado9800;\n") shard_map: Dict[str, int] = {} external_host = subprocess.check_output( [os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"], @@ -66,8 +65,9 @@ def write_updated_configs() -> None: host = f"{shard}.{external_host}" assert host not in shard_map, f"host {host} duplicated" shard_map[host] = int(port) - write_realm_nginx_config_line(nginx_sharding_conf_f, host, port) + nginx_sharding_conf_f.write(f" {nginx_quote(host)} http://tornado{port};\n") nginx_sharding_conf_f.write("\n") + nginx_sharding_conf_f.write("}\n") sharding_json_f.write(json.dumps(shard_map) + "\n") @@ -83,7 +83,7 @@ parser.add_argument( options = parser.parse_args() config_file_path = "/etc/zulip" -base_files = ["nginx_sharding.conf", "sharding.json"] +base_files = ["nginx_sharding_map.conf", "sharding.json"] full_real_paths = [f"{config_file_path}/{filename}" for filename in base_files] full_new_paths = [f"{filename}.tmp" for filename in full_real_paths] try: diff --git a/scripts/refresh-sharding-and-restart b/scripts/refresh-sharding-and-restart index 4b9ee77b7f..8fb9d9235b 100755 --- a/scripts/refresh-sharding-and-restart +++ b/scripts/refresh-sharding-and-restart @@ -7,16 +7,16 @@ set -e SUPPRESS_SHARDING_NOTICE=1 "$(dirname "$0")/zulip-puppet-apply" -f # Verify, before we move them into place -if ! [ -e /etc/zulip/nginx_sharding.conf.tmp ] || ! [ -e /etc/zulip/sharding.json.tmp ]; then +if ! [ -e /etc/zulip/nginx_sharding_map.conf.tmp ] || ! [ -e /etc/zulip/sharding.json.tmp ]; then echo "No sharding updates found to apply." exit 1 fi -chown root:root /etc/zulip/nginx_sharding.conf.tmp -chmod 644 /etc/zulip/nginx_sharding.conf.tmp +chown root:root /etc/zulip/nginx_sharding_map.conf.tmp +chmod 644 /etc/zulip/nginx_sharding_map.conf.tmp chown zulip:zulip /etc/zulip/sharding.json.tmp chmod 644 /etc/zulip/sharding.json.tmp -mv /etc/zulip/nginx_sharding.conf.tmp /etc/zulip/nginx_sharding.conf +mv /etc/zulip/nginx_sharding_map.conf.tmp /etc/zulip/nginx_sharding_map.conf mv /etc/zulip/sharding.json.tmp /etc/zulip/sharding.json # In the ordering of operations below, the crucial detail is that