sharding: Configure Tornado sharding with nginx map.

https://nginx.org/en/docs/http/ngx_http_map_module.html

Since Puppet doesn’t manage the contents of nginx_sharding.conf after
its initial creation, it needs to be renamed so we can give it
different default contents.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

puppet/zulip/manifests/tornado_sharding.pp

@@ -5,15 +5,22 @@ class zulip::tornado_sharding {
   # with the correct default content for the "only one shard" setup. For this
   # reason they use "replace => false", because the files are managed by
   # the sharding script afterwards and Puppet shouldn't overwrite them.
-  file { '/etc/zulip/nginx_sharding.conf':
+  file { '/etc/zulip/nginx_sharding_map.conf':
     ensure  => file,
     owner   => 'root',
     group   => 'root',
     mode    => '0644',
     notify  => Service['nginx'],
-    content => "set \$tornado_server http://tornado;\n",
+    content => @(EOT),
+      map "" $tornado_server {
+          default http://tornado;
+      }
+      | EOT
     replace => false,
   }
+  file { '/etc/zulip/nginx_sharding.conf':
+    ensure => absent,
+  }
   file { '/etc/zulip/sharding.json':
     ensure  => file,
     require => User['zulip'],
@@ -29,7 +36,7 @@ class zulip::tornado_sharding {
   exec { 'stage_updated_sharding':
     command   => "${::zulip_scripts_path}/lib/sharding.py",
     onlyif    => "${::zulip_scripts_path}/lib/sharding.py --errors-ok",
-    require   => [File['/etc/zulip/nginx_sharding.conf'], File['/etc/zulip/sharding.json']],
+    require   => [File['/etc/zulip/nginx_sharding_map.conf'], File['/etc/zulip/sharding.json']],
     logoutput => true,
     loglevel  => warning,
   }

puppet/zulip/templates/nginx/zulip-enterprise.template.erb

@@ -13,6 +13,7 @@ server {
 <% end -%>
 
 include /etc/nginx/zulip-include/upstreams;
+include /etc/zulip/nginx_sharding_map.conf;
 
 server {
 <% if @nginx_http_only -%>
@@ -30,7 +31,6 @@ server {
         alias /home/zulip/local-static;
     }
 
-    include /etc/zulip/nginx_sharding.conf;
     include /etc/nginx/zulip-include/certbot;
     include /etc/nginx/zulip-include/app;
 }

puppet/zulip_ops/files/nginx/sites-available/zulip

@@ -1,4 +1,5 @@
 include /etc/nginx/zulip-include/upstreams;
+include /etc/zulip/nginx_sharding_map.conf;
 
 server {
     listen 443 ssl http2;
@@ -15,6 +16,5 @@ server {
 
     server_name zulipchat.com *.zulipchat.com;
 
-    include /etc/zulip/nginx_sharding.conf;
     include /etc/nginx/zulip-include/app;
 }

puppet/zulip_ops/files/nginx/sites-available/zulip-staging

@@ -6,6 +6,7 @@ server {
 }
 
 include /etc/nginx/zulip-include/upstreams;
+include /etc/zulip/nginx_sharding_map.conf;
 
 server {
     listen 443 ssl http2;
@@ -22,6 +23,5 @@ server {
 
     server_name zulipstaging.com;
 
-    include /etc/zulip/nginx_sharding.conf;
     include /etc/nginx/zulip-include/app;
 }

scripts/lib/sharding.py

@@ -5,7 +5,7 @@ import json
 import os
 import subprocess
 import sys
-from typing import Any, Dict
+from typing import Dict
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(BASE_DIR)
@@ -16,17 +16,13 @@ setup_path()
 from scripts.lib.zulip_tools import get_config_file, get_tornado_ports
 
 
-def write_realm_nginx_config_line(f: Any, host: str, port: str) -> None:
-    f.write(
-        f"""if ($host = '{host}') {{
-    set $tornado_server http://tornado{port};
-}}\n"""
-    )
+def nginx_quote(s: str) -> str:
+    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
 
 
 # Basic system to do Tornado sharding.  Writes two output .tmp files that need
 # to be renamed to the following files to finalize the changes:
-# * /etc/zulip/nginx_sharding.conf; nginx needs to be reloaded after changing.
+# * /etc/zulip/nginx_sharding_map.conf; nginx needs to be reloaded after changing.
 # * /etc/zulip/sharding.json; supervisor Django process needs to be reloaded
 # after changing.  TODO: We can probably make this live-reload by statting the file.
 #
@@ -40,16 +36,19 @@ def write_updated_configs() -> None:
         sorted(ports) == expected_ports
     ), f"ports ({sorted(ports)}) must be contiguous, starting with 9800"
 
-    with open("/etc/zulip/nginx_sharding.conf.tmp", "w") as nginx_sharding_conf_f, open(
+    with open("/etc/zulip/nginx_sharding_map.conf.tmp", "w") as nginx_sharding_conf_f, open(
         "/etc/zulip/sharding.json.tmp", "w"
     ) as sharding_json_f:
 
         if len(ports) == 1:
-            nginx_sharding_conf_f.write("set $tornado_server http://tornado;\n")
+            nginx_sharding_conf_f.write('map "" $tornado_server {\n')
+            nginx_sharding_conf_f.write("    default http://tornado;\n")
+            nginx_sharding_conf_f.write("}\n")
             sharding_json_f.write("{}\n")
             return
 
-        nginx_sharding_conf_f.write("set $tornado_server http://tornado9800;\n")
+        nginx_sharding_conf_f.write("map $http_host $tornado_server {\n")
+        nginx_sharding_conf_f.write("    default http://tornado9800;\n")
         shard_map: Dict[str, int] = {}
         external_host = subprocess.check_output(
             [os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"],
@@ -66,8 +65,9 @@ def write_updated_configs() -> None:
                         host = f"{shard}.{external_host}"
                     assert host not in shard_map, f"host {host} duplicated"
                     shard_map[host] = int(port)
-                    write_realm_nginx_config_line(nginx_sharding_conf_f, host, port)
+                    nginx_sharding_conf_f.write(f"    {nginx_quote(host)} http://tornado{port};\n")
             nginx_sharding_conf_f.write("\n")
+        nginx_sharding_conf_f.write("}\n")
 
         sharding_json_f.write(json.dumps(shard_map) + "\n")
 
@@ -83,7 +83,7 @@ parser.add_argument(
 options = parser.parse_args()
 
 config_file_path = "/etc/zulip"
-base_files = ["nginx_sharding.conf", "sharding.json"]
+base_files = ["nginx_sharding_map.conf", "sharding.json"]
 full_real_paths = [f"{config_file_path}/{filename}" for filename in base_files]
 full_new_paths = [f"{filename}.tmp" for filename in full_real_paths]
 try:

scripts/refresh-sharding-and-restart

@@ -7,16 +7,16 @@ set -e
 SUPPRESS_SHARDING_NOTICE=1 "$(dirname "$0")/zulip-puppet-apply" -f
 
 # Verify, before we move them into place
-if ! [ -e /etc/zulip/nginx_sharding.conf.tmp ] || ! [ -e /etc/zulip/sharding.json.tmp ]; then
+if ! [ -e /etc/zulip/nginx_sharding_map.conf.tmp ] || ! [ -e /etc/zulip/sharding.json.tmp ]; then
     echo "No sharding updates found to apply."
     exit 1
 fi
 
-chown root:root /etc/zulip/nginx_sharding.conf.tmp
-chmod 644 /etc/zulip/nginx_sharding.conf.tmp
+chown root:root /etc/zulip/nginx_sharding_map.conf.tmp
+chmod 644 /etc/zulip/nginx_sharding_map.conf.tmp
 chown zulip:zulip /etc/zulip/sharding.json.tmp
 chmod 644 /etc/zulip/sharding.json.tmp
-mv /etc/zulip/nginx_sharding.conf.tmp /etc/zulip/nginx_sharding.conf
+mv /etc/zulip/nginx_sharding_map.conf.tmp /etc/zulip/nginx_sharding_map.conf
 mv /etc/zulip/sharding.json.tmp /etc/zulip/sharding.json
 
 # In the ordering of operations below, the crucial detail is that