paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-04 22:13:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

sharding: Support Tornado sharding by regexes.

Browse Source 
One should now be able to configure a regex by appending _regex to the
port number:

[tornado_sharding]
9802_regex = ^[l-p].*\.zulipchat\.com$

Signed-off-by: Anders Kaseorg <anders@zulip.com>
(cherry picked from commit 987ab741f9)
This commit is contained in:
Anders Kaseorg
2022-08-31 16:11:27 -07:00
parent 667640b447
commit db0d6f81c0
4 changed files with 41 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
scripts/lib/sharding.py
View File

@@ -31,10 +31,8 @@ def write_updated_configs() -> None:
config_file = get_config_file()
ports = get_tornado_ports(config_file)
expected_ports = list(range(9800, max(ports) + 1))
assert (
sorted(ports) == expected_ports
), f"ports ({sorted(ports)}) must be contiguous, starting with 9800"
expected_ports = list(range(9800, ports[-1] + 1))
assert ports == expected_ports, f"ports ({ports}) must be contiguous, starting with 9800"
with open("/etc/zulip/nginx_sharding_map.conf.tmp", "w") as nginx_sharding_conf_f, open(
"/etc/zulip/sharding.json.tmp", "w"
@@ -50,26 +48,33 @@ def write_updated_configs() -> None:
nginx_sharding_conf_f.write("map $http_host $tornado_server {\n")
nginx_sharding_conf_f.write(" default http://tornado9800;\n")
shard_map: Dict[str, int] = {}
shard_regexes = []
external_host = subprocess.check_output(
[os.path.join(BASE_DIR, "scripts/get-django-setting"), "EXTERNAL_HOST"],
text=True,
).strip()
for port in config_file["tornado_sharding"]:
shards = config_file["tornado_sharding"][port].strip()
if shards:
for shard in shards.split(" "):
for key, shards in config_file["tornado_sharding"].items():
if key.endswith("_regex"):
port = int(key[: -len("_regex")])
shard_regexes.append((shards, port))
nginx_sharding_conf_f.write(
f" {nginx_quote('~*' + shards)} http://tornado{port};\n"
)
else:
port = int(key)
for shard in shards.split():
if "." in shard:
host = shard
else:
host = f"{shard}.{external_host}"
assert host not in shard_map, f"host {host} duplicated"
shard_map[host] = int(port)
shard_map[host] = port
nginx_sharding_conf_f.write(f" {nginx_quote(host)} http://tornado{port};\n")
nginx_sharding_conf_f.write("\n")
nginx_sharding_conf_f.write("}\n")
sharding_json_f.write(json.dumps(shard_map) + "\n")
data = {"shard_map": shard_map, "shard_regexes": shard_regexes}
sharding_json_f.write(json.dumps(data) + "\n")
parser = argparse.ArgumentParser(