diff --git a/puppet/zulip_ops/manifests/apt_repository_debathena.pp b/puppet/zulip_ops/manifests/apt_repository_debathena.pp index e8e5b57b57..293250c513 100644 --- a/puppet/zulip_ops/manifests/apt_repository_debathena.pp +++ b/puppet/zulip_ops/manifests/apt_repository_debathena.pp @@ -1,6 +1,7 @@ class zulip_ops::apt_repository_debathena { - $setup_file = "${::zulip_scripts_path}/lib/setup-apt-repo-debathena" + $setup_apt_repo_file = "${::zulip_scripts_path}/lib/setup-apt-repo" exec { 'setup_apt_repo_debathena': - command => "bash -c '${setup_file}'", + command => "${setup_apt_repo_file} --list zulip_debathena", + unless => "${setup_apt_repo_file} --list zulip_debathena --verify", } } diff --git a/puppet/zulip_ops/manifests/ksplice_uptrack.pp b/puppet/zulip_ops/manifests/ksplice_uptrack.pp index cad901baf0..738f5799a6 100644 --- a/puppet/zulip_ops/manifests/ksplice_uptrack.pp +++ b/puppet/zulip_ops/manifests/ksplice_uptrack.pp @@ -14,10 +14,10 @@ class zulip_ops::ksplice_uptrack { mode => '0640', content => template('zulip_ops/uptrack/uptrack.conf.erb'), } - $setup_apt_repo_file = "${::zulip_scripts_path}/lib/setup-apt-repo-ksplice" + $setup_apt_repo_file = "${::zulip_scripts_path}/lib/setup-apt-repo" exec{ 'setup-apt-repo-ksplice': - command => $setup_apt_repo_file, - unless => "${setup_apt_repo_file} --verify", + command => "${setup_apt_repo_file} --list ksplice", + unless => "${setup_apt_repo_file} --list ksplice --verify", } Package { 'uptrack': require => [ diff --git a/scripts/lib/setup-apt-repo b/scripts/lib/setup-apt-repo index 0be865f3e3..cb7f3b5591 100755 --- a/scripts/lib/setup-apt-repo +++ b/scripts/lib/setup-apt-repo @@ -1,18 +1,34 @@ #!/usr/bin/env bash +# +# This script handles adding custom apt repositories into +# /etc/apt/sources.list.d/ files. It bundles the GPG keys which are +# used to verify the repositories (via `apt-key`), to explicitly pin +# the trusted signing keys, as opposed to blindly trusting HTTPS. +# +# Each /etc/apt/sources.list.d/foo.list file is created via `--list +# foo`, where `foo` defaults to `zulip`. The default `zulip.list` is +# installed in `scripts/lib/install` / `tools/lib/provision.py`, and +# other `.list` files may be installed by Puppet. set -x set -e set -u set -o pipefail verify=false -args="$(getopt -o '' --long verify -- "$@")" +args="$(getopt -o '' --long verify,list: -- "$@")" eval "set -- $args" +LIST=zulip while true; do case "$1" in --verify) verify=true shift ;; + --list) + LIST="$2" + shift + shift + ;; --) shift break @@ -23,12 +39,21 @@ done # Ensure the directory for LAST_DEPENDENCIES_HASH exists mkdir -p /var/lib/zulip -SOURCES_FILE=/etc/apt/sources.list.d/zulip.list -STAMP_FILE=/etc/apt/sources.list.d/zulip.list.apt-update-in-progress +SOURCES_FILE=/etc/apt/sources.list.d/$LIST.list +STAMP_FILE=/etc/apt/sources.list.d/$LIST.list.apt-update-in-progress -ZULIP_SCRIPTS="$(dirname "$(dirname "$0")")" -DEPENDENCIES_HASH=$(sha1sum "$ZULIP_SCRIPTS/setup/"*.asc "$0") -DEPENDENCIES_HASH_FILE="/var/lib/zulip/setup-repositories-state" +ZULIP_SCRIPTS="$(cd "$(dirname "$(dirname "$0")")" && pwd)" +LIST_PATH="$ZULIP_SCRIPTS/setup/apt-repos/$LIST" +if ! [ -d "$LIST_PATH" ]; then + echo "Not a valid value for --list: '$LIST'" + echo "" + echo "Valid values are:" + ls -1 "$ZULIP_SCRIPTS/setup/apt-repos/" + exit 1 +fi + +DEPENDENCIES_HASH=$(sha1sum "$LIST_PATH/"*.asc "$0") +DEPENDENCIES_HASH_FILE="/var/lib/zulip/setup-repositories-state-$LIST" # Ensure that DEPENDENCIES_HASH_FILE exists before hashing it. touch "$DEPENDENCIES_HASH_FILE" LAST_DEPENDENCIES_HASH="$(cat "$DEPENDENCIES_HASH_FILE")" @@ -53,66 +78,24 @@ if ! apt-get -dy install "${pre_setup_deps[@]}"; then fi apt-get -y install "${pre_setup_deps[@]}" -SCRIPTS_PATH="$(cd "$(dirname "$(dirname "$0")")" && pwd)" - release=$(lsb_release -sc) -if [[ "$release" =~ ^(bionic|cosmic|disco|eoan|focal|groovy)$ ]]; then - distribution=ubuntu - apt-key add "$SCRIPTS_PATH"/setup/pgdg.asc - apt-key add "$SCRIPTS_PATH"/setup/pgroonga-ppa.asc - cat >$SOURCES_FILE <$SOURCES_FILE <$SOURCES_FILE <"$DEPENDENCIES_HASH_FILE" diff --git a/scripts/lib/setup-apt-repo-ksplice b/scripts/lib/setup-apt-repo-ksplice deleted file mode 100755 index 34bb24e138..0000000000 --- a/scripts/lib/setup-apt-repo-ksplice +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -set -x -set -e -set -u -set -o pipefail - -verify=false -args="$(getopt -o '' --long verify -- "$@")" -eval "set -- $args" -while true; do - case "$1" in - --verify) - verify=true - shift - ;; - --) - shift - break - ;; - esac -done - -# Ensure the directory for LAST_DEPENDENCIES_HASH exists -mkdir -p /var/lib/zulip - -SOURCES_FILE=/etc/apt/sources.list.d/ksplice.list -STAMP_FILE=/etc/apt/sources.list.d/ksplice.list.apt-update-in-progress - -ZULIP_SCRIPTS="$(dirname "$(dirname "$0")")" -DEPENDENCIES_HASH=$(sha1sum "$ZULIP_SCRIPTS/setup/"*.asc "$0") -DEPENDENCIES_HASH_FILE="/var/lib/zulip/setup-repositories-state-ksplice" -# Ensure that DEPENDENCIES_HASH_FILE exists before hashing it. -touch "$DEPENDENCIES_HASH_FILE" -LAST_DEPENDENCIES_HASH="$(cat "$DEPENDENCIES_HASH_FILE")" - -# First, we only do anything in setup-apt-repo if any of its inputs -# (apt keys, code, etc.) changed. -if [ "$DEPENDENCIES_HASH" = "$LAST_DEPENDENCIES_HASH" ]; then - exit 0 -elif [ "$verify" == true ]; then - exit 1 -fi - -# Ensure that the sources file exists -touch "$SOURCES_FILE" - -# Hash it to check if the sources file is changed by the script later. -zulip_source_hash=$(sha1sum "$SOURCES_FILE") - -pre_setup_deps=(lsb-release apt-transport-https ca-certificates gnupg wget) -if ! apt-get -dy install "${pre_setup_deps[@]}"; then - apt-get update -fi -apt-get -y install "${pre_setup_deps[@]}" - -SCRIPTS_PATH="$(cd "$(dirname "$(dirname "$0")")" && pwd)" - -release=$(lsb_release -sc) -if [[ "$release" =~ ^(buster|bullseye|bionic|cosmic|disco|eoan|focal|groovy)$ ]]; then - apt-key add "$SCRIPTS_PATH"/setup/ksplice.asc - cat >$SOURCES_FILE <"$DEPENDENCIES_HASH_FILE" diff --git a/scripts/setup/apt-repos/ksplice/bionic.list b/scripts/setup/apt-repos/ksplice/bionic.list new file mode 100644 index 0000000000..ad2dfde12e --- /dev/null +++ b/scripts/setup/apt-repos/ksplice/bionic.list @@ -0,0 +1,2 @@ +deb http://www.ksplice.com/apt bionic ksplice +deb-src http://www.ksplice.com/apt bionic ksplice diff --git a/scripts/setup/apt-repos/ksplice/focal.list b/scripts/setup/apt-repos/ksplice/focal.list new file mode 100644 index 0000000000..010102a5ec --- /dev/null +++ b/scripts/setup/apt-repos/ksplice/focal.list @@ -0,0 +1,2 @@ +deb http://www.ksplice.com/apt focal ksplice +deb-src http://www.ksplice.com/apt focal ksplice diff --git a/scripts/setup/ksplice.asc b/scripts/setup/apt-repos/ksplice/ksplice.asc similarity index 100% rename from scripts/setup/ksplice.asc rename to scripts/setup/apt-repos/ksplice/ksplice.asc diff --git a/scripts/setup/apt-repos/zulip/bionic.list b/scripts/setup/apt-repos/zulip/bionic.list new file mode 100644 index 0000000000..e6be5189ca --- /dev/null +++ b/scripts/setup/apt-repos/zulip/bionic.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu bionic main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu bionic main diff --git a/scripts/setup/apt-repos/zulip/bullseye.list b/scripts/setup/apt-repos/zulip/bullseye.list new file mode 100644 index 0000000000..4c6663d160 --- /dev/null +++ b/scripts/setup/apt-repos/zulip/bullseye.list @@ -0,0 +1,2 @@ +deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main diff --git a/scripts/setup/apt-repos/zulip/buster.list b/scripts/setup/apt-repos/zulip/buster.list new file mode 100644 index 0000000000..777c6f33d7 --- /dev/null +++ b/scripts/setup/apt-repos/zulip/buster.list @@ -0,0 +1,2 @@ +deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main diff --git a/scripts/setup/apt-repos/zulip/cosmic.list b/scripts/setup/apt-repos/zulip/cosmic.list new file mode 100644 index 0000000000..e9f1e1971d --- /dev/null +++ b/scripts/setup/apt-repos/zulip/cosmic.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ cosmic-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ cosmic-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu cosmic main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu cosmic main diff --git a/scripts/setup/apt-repos/zulip/custom.sh b/scripts/setup/apt-repos/zulip/custom.sh new file mode 100755 index 0000000000..bd89cb688f --- /dev/null +++ b/scripts/setup/apt-repos/zulip/custom.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +if [[ ! -e /usr/share/doc/groonga-apt-source/copyright ]]; then + remove_pgroonga_apt_tmp_dir() { + rm -rf "$pgroonga_apt_tmp_dir" + } + pgroonga_apt_tmp_dir=$(mktemp --directory) + trap remove_pgroonga_apt_tmp_dir EXIT + + { + cd "$pgroonga_apt_tmp_dir" || exit 1 + tmp_gpg_home=.gnupg + pgroonga_apt_sign_key="$LIST_PATH/pgroonga-packages.groonga.org.asc" + gpg --homedir="$tmp_gpg_home" --import "$pgroonga_apt_sign_key" + # Find fingerprint of the first key. + pgroonga_apt_sign_key_fingerprint=$( + gpg --homedir="$tmp_gpg_home" --with-colons --list-keys \ + | grep '^fpr:' \ + | cut --delimiter=: --fields=10 \ + | head --lines=1 + ) + release=$(lsb_release -sc) + distribution=$(lsb_release -si | tr '[:upper:]' '[:lower:]') + groonga_apt_source_deb="groonga-apt-source-latest-$release.deb" + groonga_apt_source_deb_sign="$groonga_apt_source_deb.asc.$pgroonga_apt_sign_key_fingerprint" + wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb" + wget "https://packages.groonga.org/$distribution/$groonga_apt_source_deb_sign" + gpg \ + --homedir="$tmp_gpg_home" \ + --verify \ + "$groonga_apt_source_deb_sign" \ + "$groonga_apt_source_deb" + # To suppress the following warning by "apt-get install": + # N: Download is performed unsandboxed as root as file + # '.../groonga-apt-source-latest-$release.deb' couldn't be + # accessed by user '_apt'. - pkgAcquire::Run (13: Permission denied) + chown _apt . + apt-get -y install "./$groonga_apt_source_deb" + } + touch "$STAMP_FILE" +fi diff --git a/scripts/setup/apt-repos/zulip/disco.list b/scripts/setup/apt-repos/zulip/disco.list new file mode 100644 index 0000000000..1cb3e2aedb --- /dev/null +++ b/scripts/setup/apt-repos/zulip/disco.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ disco-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ disco-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu disco main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu disco main diff --git a/scripts/setup/apt-repos/zulip/eoan.list b/scripts/setup/apt-repos/zulip/eoan.list new file mode 100644 index 0000000000..3121f22e5a --- /dev/null +++ b/scripts/setup/apt-repos/zulip/eoan.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ eoan-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ eoan-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu eoan main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu eoan main diff --git a/scripts/setup/apt-repos/zulip/focal.list b/scripts/setup/apt-repos/zulip/focal.list new file mode 100644 index 0000000000..8578405cfe --- /dev/null +++ b/scripts/setup/apt-repos/zulip/focal.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu focal main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu focal main diff --git a/scripts/setup/apt-repos/zulip/groovy.list b/scripts/setup/apt-repos/zulip/groovy.list new file mode 100644 index 0000000000..5248ef84c8 --- /dev/null +++ b/scripts/setup/apt-repos/zulip/groovy.list @@ -0,0 +1,5 @@ +deb http://apt.postgresql.org/pub/repos/apt/ groovy-pgdg main +deb-src http://apt.postgresql.org/pub/repos/apt/ groovy-pgdg main + +deb http://ppa.launchpad.net/groonga/ppa/ubuntu groovy main +deb-src http://ppa.launchpad.net/groonga/ppa/ubuntu groovy main diff --git a/scripts/setup/pgdg.asc b/scripts/setup/apt-repos/zulip/pgdg.asc similarity index 100% rename from scripts/setup/pgdg.asc rename to scripts/setup/apt-repos/zulip/pgdg.asc diff --git a/scripts/setup/pgroonga-packages.groonga.org.asc b/scripts/setup/apt-repos/zulip/pgroonga-packages.groonga.org.asc similarity index 100% rename from scripts/setup/pgroonga-packages.groonga.org.asc rename to scripts/setup/apt-repos/zulip/pgroonga-packages.groonga.org.asc diff --git a/scripts/setup/pgroonga-ppa.asc b/scripts/setup/apt-repos/zulip/pgroonga-ppa.asc similarity index 100% rename from scripts/setup/pgroonga-ppa.asc rename to scripts/setup/apt-repos/zulip/pgroonga-ppa.asc diff --git a/scripts/setup/apt-repos/zulip_debathena/bionic.list b/scripts/setup/apt-repos/zulip_debathena/bionic.list new file mode 100644 index 0000000000..74cdabd83a --- /dev/null +++ b/scripts/setup/apt-repos/zulip_debathena/bionic.list @@ -0,0 +1,2 @@ +deb http://debathena.mit.edu/apt bionic debathena debathena-config +deb-src http://debathena.mit.edu/apt bionic debathena debathena-config diff --git a/scripts/setup/debathena-archive.asc b/scripts/setup/apt-repos/zulip_debathena/debathena-archive.asc similarity index 100% rename from scripts/setup/debathena-archive.asc rename to scripts/setup/apt-repos/zulip_debathena/debathena-archive.asc