zulip/scripts/lib/install

#!/usr/bin/env bash
set -e

usage() {
    # A subset of this documentation also appears in docs/production/install.md
    cat <<'EOF'
Usage:
  install --hostname=zulip.example.com --email=zulip-admin@example.com [options...]
  install --help

Options:
  --hostname=zulip.example.com
      The user-accessible domain name for this Zulip server, i.e., what users will type
      in their web browser.  Required, unless --no-init-db is set and --certbot is not.
  --email=zulip-admin@example.com
      The email address of the person or team who should get support and error emails
      from this Zulip server.  Required, unless --no-init-db is set and --certbot is
      not.

  --certbot
      Obtains a free SSL certificate for the server using Certbot,
      https://certbot.eff.org/  Recommended.  Conflicts with --self-signed-cert.
  --self-signed-cert
      Generate a self-signed SSL certificate for the server. This isn’t suitable for
      production use, but may be convenient for testing.  Conflicts with --certbot.
  --cacert=/path/to/ca.pem
      Set the CA which used to establish TLS to all public internet sites during the
      install process; used when this command is run once in a highly-controlled
      environment to produce an image which is used elsewhere.  Uncommon.

  --postgresql-version=12
      Sets the version of PostgreSQL that will be installed.
  --postgresql-missing-dictionaries
      Set postgresql.missing_dictionaries, which alters the initial database.  Use with
      cloud managed databases like RDS.  Conflicts with --no-overwrite-settings.
  --no-init-db
      Does not do any database initialization; use when you already have a Zulip
      database.

  --no-overwrite-settings
      Preserve existing `/etc/zulip` configuration files.
  --no-dist-upgrade
      Skip the initial `apt-get dist-upgrade`.

EOF
}

# Shell option parsing.  Over time, we'll want to move some of the
# environment variables below into this self-documenting system.
args="$(getopt -o '' --long help,hostname:,email:,certbot,self-signed-cert,cacert:,postgresql-version:,postgresql-missing-dictionaries,no-init-db,no-overwrite-settings,no-dist-upgrade -n "$0" -- "$@")"
eval "set -- $args"
while true; do
    case "$1" in
        --help)
            usage
            exit 0
            ;;

        --hostname)
            EXTERNAL_HOST="$2"
            shift
            shift
            ;;
        --email)
            ZULIP_ADMINISTRATOR="$2"
            shift
            shift
            ;;

        --certbot)
            USE_CERTBOT=1
            shift
            ;;
        --cacert)
            export CUSTOM_CA_CERTIFICATES="$2"
            shift
            shift
            ;;
        --self-signed-cert)
            SELF_SIGNED_CERT=1
            shift
            ;;

        --postgresql-version)
            POSTGRESQL_VERSION="$2"
            shift
            shift
            ;;
        --postgresql-missing-dictionaries)
            POSTGRESQL_MISSING_DICTIONARIES=1
            shift
            ;;
        --no-init-db)
            NO_INIT_DB=1
            shift
            ;;

        --no-overwrite-settings)
            NO_OVERWRITE_SETTINGS=1
            shift
            ;;
        --no-dist-upgrade)
            NO_DIST_UPGRADE=1
            shift
            ;;
        --)
            shift
            break
            ;;
    esac
done

if [ "$#" -gt 0 ]; then
    usage >&2
    exit 1
fi

## Options from environment variables.
#
# Specify options for apt.
read -r -a APT_OPTIONS <<<"${APT_OPTIONS:-}"
# Install additional packages.
read -r -a ADDITIONAL_PACKAGES <<<"${ADDITIONAL_PACKAGES:-}"
# Comma-separated list of Puppet manifests to install.  The default is
# zulip::profile::standalone for an all-in-one system or
# zulip::profile::docker for Docker.  Use
# e.g. zulip::profile::app_frontend for a Zulip frontend server.
PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::profile::standalone}"
VIRTUALENV_NEEDED="${VIRTUALENV_NEEDED:-yes}"
POSTGRESQL_VERSION="${POSTGRESQL_VERSION:-12}"

if [ -n "$SELF_SIGNED_CERT" ] && [ -n "$USE_CERTBOT" ]; then
    set +x
    echo "error: --self-signed-cert and --certbot are incompatible" >&2
    echo >&2
    usage >&2
    exit 1
fi

if [ -n "$POSTGRESQL_MISSING_DICTIONARIES" ] && [ -n "$NO_OVERWRITE_SETTINGS" ]; then
    set +x
    echo "error: --postgresql-missing-dictionaries and --no-overwrite-settings are incompatible" >&2
    echo >&2
    usage >&2
    exit 1
fi

if [ -z "$EXTERNAL_HOST" ] || [ -z "$ZULIP_ADMINISTRATOR" ]; then
    if [ -n "$USE_CERTBOT" ] || [ -z "$NO_INIT_DB" ]; then
        usage >&2
        exit 1
    fi
fi

if [ "$EXTERNAL_HOST" = zulip.example.com ] \
    || [ "$ZULIP_ADMINISTRATOR" = zulip-admin@example.com ]; then
    # These example values are specifically checked for and would fail
    # later; see check_config in zerver/lib/management.py.
    echo 'error: The example hostname and email must be replaced with real values.' >&2
    echo >&2
    usage >&2
    exit 1
fi

# Do set -x after option parsing is complete
set -x

ZULIP_PATH="$(readlink -f "$(dirname "$0")"/../..)"

# Force a known locale.  Some packages on PyPI fail to install in some locales.
localedef -i en_US -f UTF-8 en_US.UTF-8
export LC_ALL="en_US.UTF-8"
export LANG="en_US.UTF-8"
export LANGUAGE="en_US.UTF-8"

# Check for a supported OS release.
if [ -f /etc/os-release ]; then
    os_info="$(
        . /etc/os-release
        printf '%s\n' "$ID" "$ID_LIKE" "$VERSION_ID" "$VERSION_CODENAME"
    )"
    {
        read -r os_id
        read -r os_id_like
        read -r os_version_id
        read -r os_version_codename || true
    } <<<"$os_info"
    case " $os_id $os_id_like " in
        *' debian '*)
            package_system="apt"
            ;;
        *' rhel '*)
            package_system="yum"
            ;;
    esac
fi

case "$os_id$os_version_id" in
    debian10 | ubuntu18.04 | ubuntu20.04) ;;
    *)
        set +x
        cat <<EOF

Unsupported OS release: $os_id $os_version_id

Zulip in production is supported only on:
 - Debian 10 "buster"
 - Ubuntu 18.04 LTS "bionic"
 - Ubuntu 20.04 LTS "focal"

For more information, see:
  https://zulip.readthedocs.io/en/latest/production/requirements.html
EOF
        exit 1
        ;;
esac

if [ "$os_id" = ubuntu ] && ! apt-cache policy \
    | grep -q "^     release v=$os_version_id,o=Ubuntu,a=$os_version_codename,n=$os_version_codename,l=Ubuntu,c=universe"; then
    set +x
    cat <<'EOF'

You must enable the Ubuntu Universe repository before installing
Zulip.  You can do this with:

    sudo add-apt-repository universe
    sudo apt update

For more information, see:
  https://zulip.readthedocs.io/en/latest/production/requirements.html
EOF
    exit 1
fi

case ",$PUPPET_CLASSES," in
    *,zulip::profile::standalone,* | *,zulip::profile::postgresql,*)
        if [ "$package_system" = apt ]; then
            # We're going to install PostgreSQL from the PostgreSQL apt
            # repository; this may conflict with the existing PostgreSQL.
            OTHER_PG="$(dpkg --get-selections \
                | grep -E '^postgresql-[0-9]+\s+install$' \
                | grep -v "^postgresql-$POSTGRESQL_VERSION\b" \
                | cut -f 1)" || true
            if [ -n "$OTHER_PG" ]; then
                INDENTED="${OTHER_PG//$'\n'/$'\n'    }"
                SPACED="${OTHER_PG//$'\n'/ }"
                cat <<EOF

The following PostgreSQL servers were found to already be installed:

    $INDENTED

Zulip needs to install PostgreSQL $POSTGRESQL_VERSION, but does not wish
to uninstall existing databases in order to do so.  Remove all other
PostgreSQL servers manually before running the installer:

    sudo apt-get remove $SPACED

EOF
                exit 1
            fi
        fi
        ;;
esac

# Check for at least ~1.86GB of RAM before starting installation;
# otherwise users will find out about insufficient RAM via weird
# errors like a segfault running `pip install`.
# Additionally, some AWS images that are advertised to be 2 GB
# are actually 1880000B in size.
mem_kb=$(head -n1 /proc/meminfo | awk '{print $2}')
if [ "$mem_kb" -lt 1860000 ]; then
    set +x
    echo -e '\033[0;31m' >&2
    echo "Insufficient RAM.  Zulip requires at least 2GB of RAM." >&2
    echo >&2
    echo -e '\033[0m' >&2
    exit 1
fi

# Do package update, e.g. do `apt-get update` on Debian
if [ "$package_system" = apt ]; then
    # setup-apt-repo does an `apt-get update`
    "$ZULIP_PATH"/scripts/lib/setup-apt-repo
elif [ "$package_system" = yum ]; then
    "$ZULIP_PATH"/scripts/lib/setup-yum-repo
fi

# Check early for missing SSL certificates
if [ "$PUPPET_CLASSES" = "zulip::profile::standalone" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then
    set +x
    cat <<EOF

No SSL certificate found.  One or both required files is missing:
    /etc/ssl/private/zulip.key
    /etc/ssl/certs/zulip.combined-chain.crt

Suggested solutions:
 * For most sites, the --certbot option is recommended.
 * If you have your own key and cert, see docs linked below
   for how to install them.
 * For non-production testing, try the --self-signed-cert option.

For help and more details, see our SSL documentation:
  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html

Once fixed, just rerun scripts/setup/install; it'll pick up from here!

EOF
    exit 1
fi

# don't run dist-upgrade in one click apps to make the
# installation process more seamless.
if [ -z "$NO_DIST_UPGRADE" ]; then
    if [ "$package_system" = apt ]; then
        apt-get -y dist-upgrade "${APT_OPTIONS[@]}"
    elif [ "$package_system" = yum ]; then
        # On CentOS, there is no need to do `yum -y upgrade` because `yum -y
        # update` already does the same thing.
        :
    fi
fi

if [ "$package_system" = apt ]; then
    if ! apt-get install -y \
        puppet git curl wget jq \
        python3 crudini \
        "${ADDITIONAL_PACKAGES[@]}"; then
        set +x
        echo -e '\033[0;31m' >&2
        echo "Installing packages failed; is network working and (on Ubuntu) the universe repository enabled?" >&2
        echo >&2
        echo -e '\033[0m' >&2
        exit 1
    fi
elif [ "$package_system" = yum ]; then
    if ! yum install -y \
        puppet git curl wget jq \
        python3 crudini \
        "${ADDITIONAL_PACKAGES[@]}"; then
        set +x
        echo -e '\033[0;31m' >&2
        echo "Installing packages failed; is network working?" >&2
        echo >&2
        echo -e '\033[0m' >&2
        exit 1
    fi
fi

if [ -n "$USE_CERTBOT" ]; then
    "$ZULIP_PATH"/scripts/setup/setup-certbot \
        --no-zulip-conf --method=standalone \
        "$EXTERNAL_HOST" --email "$ZULIP_ADMINISTRATOR"
elif [ -n "$SELF_SIGNED_CERT" ]; then
    "$ZULIP_PATH"/scripts/setup/generate-self-signed-cert \
        --exists-ok "${EXTERNAL_HOST:-$(hostname)}"
fi

# Create and activate a virtualenv
if [ "$VIRTUALENV_NEEDED" = "yes" ]; then
    "$ZULIP_PATH"/scripts/lib/create-production-venv "$ZULIP_PATH"
    "$ZULIP_PATH"/scripts/lib/create-thumbor-venv "$ZULIP_PATH"
fi

"$ZULIP_PATH"/scripts/lib/install-node

# Generate /etc/zulip/zulip.conf .
mkdir -p /etc/zulip
has_class() {
    grep -qx "$1" /var/lib/puppet/classes.txt
}

# puppet apply --noop fails unless the user that it _would_ chown
# files to exists; https://tickets.puppetlabs.com/browse/PUP-3907
#
# The home directory here should match what's declared in base.pp.
id -u zulip &>/dev/null || useradd -m zulip --home-dir /home/zulip
if [ -n "$NO_OVERWRITE_SETTINGS" ] && [ -e "/etc/zulip/zulip.conf" ]; then
    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
        --write-catalog-summary \
        --classfile=/var/lib/puppet/classes.txt \
        >/dev/null
else
    # Write out more than we need, and remove sections that are not
    # applicable to the classes that are actually necessary.
    cat <<EOF >/etc/zulip/zulip.conf
[machine]
puppet_classes = $PUPPET_CLASSES
deploy_type = production

[postgresql]
version = $POSTGRESQL_VERSION
EOF

    if [ -n "$USE_CERTBOT" ]; then
        crudini --set /etc/zulip/zulip.conf certbot auto_renew yes
    fi

    if [ -n "$POSTGRESQL_MISSING_DICTIONARIES" ]; then
        crudini --set /etc/zulip/zulip.conf postgresql missing_dictionaries true
    fi

    "$ZULIP_PATH"/scripts/zulip-puppet-apply --noop \
        --write-catalog-summary \
        --classfile=/var/lib/puppet/classes.txt \
        >/dev/null

    # We only need the PostgreSQL version setting on database hosts; but
    # we don't know if this is a database host until we have the catalog summary.
    if ! has_class "zulip::postgresql_common" || [ "$package_system" != apt ]; then
        crudini --del /etc/zulip/zulip.conf postgresql
    fi

    # Note: there are four dpkg-query outputs to consider:
    #
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # root@host# apt install rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # install ok installed
    # root@host# apt remove rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # deinstall ok config-files
    # root@host# apt purge rabbitmq-server
    # root@host# dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null
    # unknown ok not-installed
    #
    # (There are more possibilities in the case of dpkg errors.)  Here
    # we are checking for either empty or not-installed.
    if ! dpkg-query --showformat '${Status}\n' -W rabbitmq-server 2>/dev/null | grep -vq ' not-installed$'; then
        cat <<EOF >>/etc/zulip/zulip.conf

[rabbitmq]
nodename = zulip@localhost
EOF
    fi
fi

if has_class "zulip::app_frontend_base"; then
    if [ -z "$NO_OVERWRITE_SETTINGS" ] || ! [ -e "/etc/zulip/settings.py" ]; then
        cp -a "$ZULIP_PATH"/zproject/prod_settings_template.py /etc/zulip/settings.py
        if [ -n "$EXTERNAL_HOST" ]; then
            sed -i "s/^EXTERNAL_HOST =.*/EXTERNAL_HOST = '$EXTERNAL_HOST'/" /etc/zulip/settings.py
        fi
        if [ -n "$ZULIP_ADMINISTRATOR" ]; then
            sed -i "s/^ZULIP_ADMINISTRATOR =.*/ZULIP_ADMINISTRATOR = '$ZULIP_ADMINISTRATOR'/" /etc/zulip/settings.py
        fi
    fi
    ln -nsf /etc/zulip/settings.py "$ZULIP_PATH"/zproject/prod_settings.py
    "$ZULIP_PATH"/scripts/setup/generate_secrets.py --production
fi

"$ZULIP_PATH"/scripts/zulip-puppet-apply -f

if [ "$package_system" = apt ]; then
    apt-get -y upgrade
elif [ "$package_system" = yum ]; then
    # No action is required because `yum update` already does upgrade.
    :
fi

if has_class "zulip::nginx" && ! has_class "zulip::profile::docker"; then
    # Check nginx was configured properly now that we've installed it.
    # Most common failure mode is certs not having been installed.
    if ! nginx -t; then
        (
            set +x
            cat <<EOF

Verifying the Zulip nginx configuration failed!

This is almost always a problem with your SSL certificates.  See:
  https://zulip.readthedocs.io/en/latest/production/ssl-certificates.html

Once fixed, just rerun scripts/setup/install; it'll pick up from here!

EOF
            exit 1
        )
    fi
fi

if has_class "zulip::profile::rabbitmq"; then
    if ! rabbitmqctl status >/dev/null; then
        set +x
        cat <<EOF

RabbitMQ seems to not have started properly after the installation process.
Often this is caused by misconfigured /etc/hosts in virtualized environments.
For more information, see:
  https://github.com/zulip/zulip/issues/53#issuecomment-143805121

EOF
        exit 1
    fi
    "$ZULIP_PATH"/scripts/setup/configure-rabbitmq
fi

if has_class "zulip::postgresql_common" && [ -z "$NO_INIT_DB" ]; then
    "$ZULIP_PATH"/scripts/setup/postgresql-init-db
fi

if has_class "zulip::app_frontend_base"; then
    deploy_path=$("$ZULIP_PATH"/scripts/lib/zulip_tools.py make_deploy_path)
    mv "$ZULIP_PATH" "$deploy_path"
    ln -nsf /home/zulip/deployments/next "$ZULIP_PATH"
    ln -nsf "$deploy_path" /home/zulip/deployments/next
    ln -nsf "$deploy_path" /home/zulip/deployments/current
    ln -nsf /etc/zulip/settings.py "$deploy_path"/zproject/prod_settings.py
    mkdir -p "$deploy_path"/prod-static/serve
    cp -rT "$deploy_path"/prod-static/serve /home/zulip/prod-static
    chown -R zulip:zulip /home/zulip /var/log/zulip /etc/zulip/settings.py

    if ! [ -e "/home/zulip/prod-static/generated" ]; then
        # If we're installing from a Git checkout, we need to run
        # `tools/update-prod-static` in order to build the static
        # assets.
        su zulip -c '/home/zulip/deployments/current/tools/update-prod-static'
    fi
fi

if [ -n "$NO_INIT_DB" ]; then
    set +x
    cat <<EOF

 Success!

 Stopping because --no-init-db was passed.
 To complete the installation, configure PostgreSQL and then run:

   su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database'
   su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
EOF
    exit 0
fi

su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database --quiet'

su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'