zulip/scripts/lib/upgrade-zulip-stage-2

#!/usr/bin/env python3
#
# This script contains the actual logic for upgrading from an old
# version of Zulip to the new version.  upgrade-zulip-stage-2 is
# always run from the new version of Zulip, so any bug fixes take
# effect on the very next upgrade.
import argparse
import glob
import hashlib
import logging
import os
import subprocess
import sys
import time
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import NoReturn

os.environ["PYTHONUNBUFFERED"] = "y"

# Force a known locale.  Some packages on PyPI fail to install in some locales.
os.environ["LC_ALL"] = "en_US.UTF-8"
os.environ["LANG"] = "en_US.UTF-8"
os.environ["LANGUAGE"] = "en_US.UTF-8"

sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
from scripts.lib.zulip_tools import (
    DEPLOYMENTS_DIR,
    assert_running_as_root,
    get_config,
    get_config_file,
    parse_os_release,
    set_config,
    su_to_zulip,
)

assert_running_as_root()

logging.Formatter.converter = time.gmtime
logging.basicConfig(format="%(asctime)s upgrade-zulip-stage-2: %(message)s",
                    level=logging.INFO)


def error_desupported_os(vendor: str, os_version: str) -> "NoReturn":
    # Link to documentation for how to correctly upgrade the OS.
    logging.critical("Unsupported platform: %s %s", vendor, os_version)
    logging.info("Sorry! The support for your OS has been discontinued.\n"
                 "Please upgrade your OS to a supported release first.\n"
                 "See https://zulip.readthedocs.io/en/latest/production/"
                 "upgrade-or-modify.html#upgrading-the-operating-system")
    sys.exit(1)


# Do not upgrade on unsupported OS versions.
UNSUPPORTED_DISTROS = [
    ('ubuntu', '14.04'),
    ('ubuntu', '16.04'),
    ('debian', '9'),
]
distro_info = parse_os_release()
vendor = distro_info['ID']
os_version = distro_info['VERSION_ID']

if (vendor, os_version) in UNSUPPORTED_DISTROS:
    error_desupported_os(vendor, os_version)

# make sure we have appropriate file permissions
os.umask(0o22)

parser = argparse.ArgumentParser()
parser.add_argument("deploy_path", metavar="deploy_path",
                    help="Path to deployment directory")
parser.add_argument("--skip-puppet", dest="skip_puppet", action='store_true',
                    help="Skip doing puppet/apt upgrades.")
parser.add_argument("--skip-migrations", dest="skip_migrations", action='store_true',
                    help="Skip doing migrations.")
parser.add_argument("--from-git", dest="from_git", action='store_true',
                    help="Upgrading from git, so run update-prod-static.")
parser.add_argument("--ignore-static-assets", dest="ignore_static_assets", action='store_true',
                    help="Do not attempt to copy/manage static assets.")
parser.add_argument("--skip-purge-old-deployments", dest="skip_purge_old_deployments",
                    action="store_true", help="Skip purging old deployments.")
parser.add_argument("--audit-fts-indexes", dest="audit_fts_indexes",
                    action="store_true", help="Audit and fix full text search indexes.")
args = parser.parse_args()

deploy_path = args.deploy_path
os.chdir(deploy_path)

config_file = get_config_file()
tornado_processes = int(get_config(config_file, 'application_server', 'tornado_processes', '1'))

IS_SERVER_UP = True

def shutdown_server() -> None:
    global IS_SERVER_UP

    core_server_services = ["zulip-django",
                            "zulip-tornado" if tornado_processes == 1 else "zulip-tornado:*"]
    worker_services = ["zulip-workers:*"]
    # Stop and start thumbor service only if thumbor is installed.
    if os.path.exists("/etc/supervisor/conf.d/thumbor.conf"):
        core_server_services.append("zulip-thumbor")

    logging.info("Stopping Zulip...")
    subprocess.check_call(["supervisorctl", "stop"] + core_server_services + worker_services,
                          preexec_fn=su_to_zulip)
    IS_SERVER_UP = False

# postgresql.version is required for database servers, but wasn't
# previously; fill it in based on what the OS provides.
if os.path.exists('/etc/init.d/postgresql'):
    postgres_version = get_config(config_file, 'postgresql', 'version')
    if not postgres_version:
        default_postgres_version = {
            ('debian', '10'): '11',
            ('ubuntu', '18.04'): '10',
            ('ubuntu', '20.04'): '12',
            ('centos', '7'): '11',
        }
        if (vendor, os_version) in default_postgres_version:
            postgres_version = default_postgres_version[(vendor, os_version)]
        else:
            error_desupported_os(vendor, os_version)
        set_config(config_file, 'postgresql', 'version', postgres_version)
        with open('/etc/zulip/zulip.conf', 'w') as zulipconf:
            config_file.write(zulipconf)

if glob.glob("/usr/share/postgresql/*/extension/tsearch_extras.control"):
    # Remove legacy tsearch_extras package references
    subprocess.check_call([
        "su", "postgres", "-c",
        'psql -v ON_ERROR_STOP=1 zulip -c "DROP EXTENSION IF EXISTS tsearch_extras;"'])
    subprocess.check_call(["apt-get", "remove", "-y", "postgresql-*-tsearch-extras"])

if not args.skip_puppet:
    logging.info("Upgrading system packages...")
    subprocess.check_call(["apt-get", "update"])
    subprocess.check_call(["apt-get", "-y", "upgrade"])

if not os.path.exists(os.path.join(deploy_path, "zproject/prod_settings.py")):
    # This is normally done in unpack-zulip, but for upgrading from
    # zulip<1.4.0, we need to do it.  See discussion in commit 586b23637.
    os.symlink("/etc/zulip/settings.py",
               os.path.join(deploy_path, "zproject/prod_settings.py"))

# Now we should have an environment setup where we can run our tools;
# first, creating the production venv.
subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-production-venv"),
                       deploy_path])

# Setup the thumbor venv
subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "create-thumbor-venv"),
                       deploy_path])

# Make sure the right version of node is installed
subprocess.check_call([os.path.join(deploy_path, "scripts", "lib", "install-node"),
                       deploy_path])

# Generate any new secrets that were added in the new version required.
# TODO: Do caching to only run this when it has changed.
subprocess.check_call([os.path.join(deploy_path, "scripts", "setup", "generate_secrets.py"),
                       "--production"])

# Unpleasant migration: Remove any legacy deployed copies of
# images-google-64 from before we renamed that emojiset to
# "googleblob":
emoji_path = "/home/zulip/prod-static/generated/emoji/images-google-64/1f32d.png"
if os.path.exists(emoji_path):
    with open(emoji_path, "rb") as f:
        emoji_data = f.read()
    emoji_sha = hashlib.sha1(emoji_data).hexdigest()
    if emoji_sha == "47033121dc20b376e0f86f4916969872ad22a293":
        import shutil
        shutil.rmtree("/home/zulip/prod-static/generated/emoji/images-google-64")

# And then, building/installing the static assets.
if args.ignore_static_assets:
    # For the OS version upgrade use case, the static assets are
    # already in place, and we don't need to do anything.  Further,
    # neither of the options below will work for all installations,
    # because if we installed from Git, `prod-static/serve` may be
    # empty so we can't do the non-Git thing, whereas if we installed
    # from a tarball, we won't have a `tools/` directory and thus
    # cannot run `tools/update-prod-static`.
    pass
elif args.from_git:
    # Because `upgrade-zulip-from-git` needs to build static assets, it
    # is at risk of being OOM killed on systems with limited free RAM.
    mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
    mem_gib = mem_bytes/(1024.**3)  # e.g. 3.74

    # Ideally, we'd have 2 thresholds here, depending on whether the
    # system is running queue workers multithreaded or multiprocess.
    # See puppet/zulip/manifests/app_frontend_base.pp for background.
    if mem_gib < 3.8:
        logging.info("Shutting down server to build static assets on a low-RAM system.")
        shutdown_server()

    # Note: The fact that this is before we apply puppet changes means
    # that we don't support adding new puppet dependencies of
    # update-prod-static with the git upgrade process.  But it'll fail
    # safely; this seems like a worthwhile tradeoff to minimize downtime.
    logging.info("Building static assets...")
    subprocess.check_call(["./tools/update-prod-static", "--prev-deploy",
                           os.path.join(DEPLOYMENTS_DIR, 'current')],
                          preexec_fn=su_to_zulip)
    logging.info("Caching zulip git version...")
    subprocess.check_call(["./tools/cache-zulip-git-version"], preexec_fn=su_to_zulip)
else:
    # Since this doesn't do any actual work, it's likely safe to have
    # this run before we apply puppet changes (saving a bit of downtime).
    logging.info("Installing static assets...")
    subprocess.check_call(["cp", "-rT", os.path.join(deploy_path, 'prod-static/serve'),
                           '/home/zulip/prod-static'], preexec_fn=su_to_zulip)

usermessage_index_migrations = [
    "[ ] 0082_index_starred_user_messages",
    "[ ] 0083_index_mentioned_user_messages",
    "[ ] 0095_index_unread_user_messages",
    "[ ] 0098_index_has_alert_word_user_messages",
    "[ ] 0099_index_wildcard_mentioned_user_messages",
    "[ ] 0177_user_message_add_and_index_is_private_flag",
    "[ ] 0180_usermessage_add_active_mobile_push_notification",
]
# Our next optimization is to check whether any migrations are needed
# before we start the critical section of the restart.  This saves
# about 1s of downtime in a no-op upgrade.
migrations_needed = False
if not args.skip_migrations:
    logging.info("Checking for needed migrations")
    migrations_output = subprocess.check_output(["./manage.py", "showmigrations"],
                                                preexec_fn=su_to_zulip).decode("utf-8")
    need_create_large_indexes = False
    for ln in migrations_output.split("\n"):
        line_str = ln.strip()
        if line_str.startswith("[ ]"):
            migrations_needed = True
            if line_str in usermessage_index_migrations:
                need_create_large_indexes = True
    if need_create_large_indexes:
        logging.info("Creating some expensive indexes before starting downtime.")
        subprocess.check_call(["./manage.py", "create_large_indexes"],
                              preexec_fn=su_to_zulip)

# Now we start shutting down services; we start with
# process-fts-updates, which isn't on the critical serving path.
if os.path.exists("/etc/supervisor/conf.d/zulip_db.conf"):
    subprocess.check_call(["supervisorctl", "stop", "process-fts-updates"], preexec_fn=su_to_zulip)

if (not args.skip_puppet or migrations_needed) and IS_SERVER_UP:
    # By default, we shut down the service to apply migrations and
    # puppet changes, to minimize risk of issues due to inconsistent
    # state.
    shutdown_server()

if not args.skip_puppet:
    logging.info("Applying puppet changes...")
    subprocess.check_call(["./scripts/zulip-puppet-apply", "--force"])
    subprocess.check_call(["apt-get", "-y", "upgrade"])

if migrations_needed:
    logging.info("Applying database migrations...")
    subprocess.check_call(["./manage.py", "migrate", "--noinput"], preexec_fn=su_to_zulip)

subprocess.check_call(["./manage.py", "create_realm_internal_bots"], preexec_fn=su_to_zulip)

logging.info("Restarting Zulip...")
subprocess.check_output(["./scripts/restart-server", "--fill-cache"], preexec_fn=su_to_zulip)
logging.info("Upgrade complete!")

if args.audit_fts_indexes:
    logging.info("Correcting full-text search indexes for updated dictionary files")
    logging.info("This may take a while but the server should work while it runs.")
    subprocess.check_call(["./manage.py", "audit_fts_indexes"], preexec_fn=su_to_zulip)

if not args.skip_purge_old_deployments:
    logging.info("Purging old deployments...")
    subprocess.check_call(["./scripts/purge-old-deployments"])
else:
    logging.info("Skipping purging old deployments.")