From c2185a81d6dfd172aa6cd8af969b04cb8fd56257 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 19 Oct 2020 17:49:54 -0700 Subject: [PATCH] puppet: Move top-level zulip deployments into "profile" directory. This moves the puppet configuration closer to the "roles and profiles method"[1] which is suggested for organizing puppet classes. Notably, here it makes clear which classes are meant to be able to stand alone as deployments. Shims are left behind at the previous names, for compatibility with existing `zulip.conf` files when upgrading. [1] https://puppet.com/docs/pe/2019.8/the_roles_and_profiles_method --- docs/overview/directory-structure.md | 2 +- docs/production/authentication-methods.md | 2 +- docs/production/deployment.md | 25 ++-- docs/production/email-gateway.md | 2 +- puppet/zulip/manifests/app_frontend.pp | 58 +------- puppet/zulip/manifests/base.pp | 138 +----------------- puppet/zulip/manifests/dockervoyager.pp | 27 +--- puppet/zulip/manifests/memcached.pp | 107 +------------- puppet/zulip/manifests/nagios.pp | 2 +- .../zulip/manifests/postgres_appdb_tuned.pp | 42 +----- .../zulip/manifests/profile/app_frontend.pp | 58 ++++++++ puppet/zulip/manifests/profile/base.pp | 138 ++++++++++++++++++ .../zulip/manifests/profile/dockervoyager.pp | 27 ++++ puppet/zulip/manifests/profile/memcached.pp | 107 ++++++++++++++ .../manifests/profile/postgres_appdb_tuned.pp | 42 ++++++ puppet/zulip/manifests/profile/rabbit.pp | 75 ++++++++++ puppet/zulip/manifests/profile/redis.pp | 68 +++++++++ puppet/zulip/manifests/profile/thumbor.pp | 25 ++++ puppet/zulip/manifests/profile/voyager.pp | 22 +++ puppet/zulip/manifests/rabbit.pp | 75 +--------- puppet/zulip/manifests/redis.pp | 68 +-------- puppet/zulip/manifests/thumbor.pp | 25 +--- puppet/zulip/manifests/voyager.pp | 24 +-- .../10/postgresql.conf.centos.template.erb | 8 +- .../10/postgresql.conf.template.erb | 8 +- .../11/postgresql.conf.template.erb | 8 +- .../12/postgresql.conf.template.erb | 8 +- .../9.5/postgresql.conf.template.erb | 8 +- .../9.6/postgresql.conf.template.erb | 8 +- puppet/zulip_ops/manifests/app_frontend.pp | 4 +- puppet/zulip_ops/manifests/base.pp | 2 +- puppet/zulip_ops/manifests/postgres_appdb.pp | 2 +- puppet/zulip_ops/manifests/redis.pp | 2 +- scripts/lib/install | 15 +- scripts/setup/upgrade-postgres | 2 +- tools/ci/production-verify | 2 +- tools/setup/install-aws-server | 4 +- 37 files changed, 638 insertions(+), 602 deletions(-) create mode 100644 puppet/zulip/manifests/profile/app_frontend.pp create mode 100644 puppet/zulip/manifests/profile/base.pp create mode 100644 puppet/zulip/manifests/profile/dockervoyager.pp create mode 100644 puppet/zulip/manifests/profile/memcached.pp create mode 100644 puppet/zulip/manifests/profile/postgres_appdb_tuned.pp create mode 100644 puppet/zulip/manifests/profile/rabbit.pp create mode 100644 puppet/zulip/manifests/profile/redis.pp create mode 100644 puppet/zulip/manifests/profile/thumbor.pp create mode 100644 puppet/zulip/manifests/profile/voyager.pp diff --git a/docs/overview/directory-structure.md b/docs/overview/directory-structure.md index b47e01d6e4..94054a571c 100644 --- a/docs/overview/directory-structure.md +++ b/docs/overview/directory-structure.md @@ -144,7 +144,7 @@ This is used to deploy essentially all configuration in production. * `puppet/zulip/` For configuration for production deployments. -* `puppet/zulip/manifests/voyager.pp` Main manifest for Zulip standalone deployments. +* `puppet/zulip/manifests/profile/voyager.pp` Main manifest for Zulip standalone deployments. ----------------------------------------------------------------------- diff --git a/docs/production/authentication-methods.md b/docs/production/authentication-methods.md index 7c4c4a5530..c966f364db 100644 --- a/docs/production/authentication-methods.md +++ b/docs/production/authentication-methods.md @@ -481,7 +481,7 @@ straightforward way to deploy that SSO solution with Zulip. 2. Edit `/etc/zulip/zulip.conf` and change the `puppet_classes` line to read: ``` - puppet_classes = zulip::voyager, zulip::apache_sso + puppet_classes = zulip::profile::voyager, zulip::apache_sso ``` 3. As root, run `/home/zulip/deployments/current/scripts/zulip-puppet-apply` diff --git a/docs/production/deployment.md b/docs/production/deployment.md index 9d820704b7..11ca9f1598 100644 --- a/docs/production/deployment.md +++ b/docs/production/deployment.md @@ -60,22 +60,21 @@ itself (e.g. installing our Postgres extensions), we have designed the Puppet configuration that Zulip uses for installing and upgrading configuration to be completely modular. -For example, you can install a Zulip rabbitmq server on a machine, you -can do the following after unpacking a Zulip production release -tarball: +For example, to install a Zulip Redis server on a machine, you can run +the following after unpacking a Zulip production release tarball: ``` -env PUPPET_CLASSES=zulip::base,zulip::redis ./scripts/setup/install +env PUPPET_CLASSES=zulip::profile::redis ./scripts/setup/install ``` -You can see most likely manifests you might want to choose in the list -of includes in -[the main manifest for the default all-in-one Zulip server][voyager.pp], -though it's also possible to subclass some of the lower-level -manifests defined in that directory if you want to customize. A good -example of doing this is in the -[zulip_ops Puppet configuration][zulipchat-puppet] that we use as part -of managing chat.zulip.org and zulip.com. +All puppet modules under `zulip::profile` are allowed to be configured +stand-alone on a host. You can see most likely manifests you might +want to choose in the list of includes in [the main manifest for the +default all-in-one Zulip server][voyager.pp], though it's also +possible to subclass some of the lower-level manifests defined in that +directory if you want to customize. A good example of doing this is +in the [zulip_ops Puppet configuration][zulipchat-puppet] that we use +as part of managing chat.zulip.org and zulip.com. ### Using Zulip with Amazon RDS as the database @@ -262,7 +261,7 @@ your installation. [nginx-proxy-config]: https://github.com/zulip/zulip/blob/master/puppet/zulip/files/nginx/zulip-include-common/proxy [nginx-proxy-longpolling-config]: https://github.com/zulip/zulip/blob/master/puppet/zulip/files/nginx/zulip-include-common/proxy_longpolling -[voyager.pp]: https://github.com/zulip/zulip/blob/master/puppet/zulip/manifests/voyager.pp +[voyager.pp]: https://github.com/zulip/zulip/blob/master/puppet/zulip/manifests/profile/voyager.pp [zulipchat-puppet]: https://github.com/zulip/zulip/tree/master/puppet/zulip_ops/manifests [nginx-loadbalancer]: https://github.com/zulip/zulip/blob/master/puppet/zulip_ops/files/nginx/sites-available/loadbalancer diff --git a/docs/production/email-gateway.md b/docs/production/email-gateway.md index 73fb1820d9..962ac737e1 100644 --- a/docs/production/email-gateway.md +++ b/docs/production/email-gateway.md @@ -58,7 +58,7 @@ using an [HTTP reverse proxy][reverse-proxy]). 1. Add `, zulip::postfix_localmail` to `puppet_classes` in `/etc/zulip/zulip.conf`. A typical value after this change is: ``` - puppet_classes = zulip::voyager, zulip::postfix_localmail + puppet_classes = zulip::profile::voyager, zulip::postfix_localmail ``` 1. If `hostname.example.com` is different from diff --git a/puppet/zulip/manifests/app_frontend.pp b/puppet/zulip/manifests/app_frontend.pp index 6c88f6a32e..d810e40190 100644 --- a/puppet/zulip/manifests/app_frontend.pp +++ b/puppet/zulip/manifests/app_frontend.pp @@ -1,58 +1,4 @@ -# Default configuration for a Zulip app frontend +# @summary Temporary shim for app frontend profile class zulip::app_frontend { - include zulip::base - include zulip::app_frontend_base - include zulip::app_frontend_once - - $nginx_http_only = zulipconf('application_server', 'http_only', undef) - if $nginx_http_only != '' { - $nginx_listen_port = zulipconf('application_server', 'nginx_listen_port', 80) - } else { - $nginx_listen_port = zulipconf('application_server', 'nginx_listen_port', 443) - } - $no_serve_uploads = zulipconf('application_server', 'no_serve_uploads', undef) - $ssl_dir = $::osfamily ? { - 'debian' => '/etc/ssl', - 'redhat' => '/etc/pki/tls', - } - file { '/etc/nginx/sites-available/zulip-enterprise': - ensure => file, - require => Package[$zulip::common::nginx], - owner => 'root', - group => 'root', - mode => '0644', - content => template('zulip/nginx/zulip-enterprise.template.erb'), - notify => Service['nginx'], - } - file { '/etc/logrotate.d/zulip': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/logrotate/zulip', - } - file { '/etc/nginx/sites-enabled/zulip-enterprise': - ensure => 'link', - require => Package[$zulip::common::nginx], - target => '/etc/nginx/sites-available/zulip-enterprise', - notify => Service['nginx'], - } - - # Trigger 2x a day certbot renew - file { '/etc/cron.d/certbot-renew': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/certbot-renew', - } - - # Restart the server regularly to avoid potential memory leak problems. - file { '/etc/cron.d/restart-zulip': - ensure => file, - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/cron.d/restart-zulip', - } + include zulip::profile::app_frontend } diff --git a/puppet/zulip/manifests/base.pp b/puppet/zulip/manifests/base.pp index 8c0265211e..4b94395a8b 100644 --- a/puppet/zulip/manifests/base.pp +++ b/puppet/zulip/manifests/base.pp @@ -1,138 +1,6 @@ -# @summary Included only by classes that can be deployed. +# @summary Temporary shim for base profile. # -# This class should only be included by classes that are intended to -# be able to be deployed on their own host. +# Any explicit PUPPET_CLASSES of this file can be removed. class zulip::base { - include zulip::common - case $::osfamily { - 'debian': { - include zulip::apt_repository - } - 'redhat': { - include zulip::yum_repository - } - default: { - fail('osfamily not supported') - } - } - case $::osfamily { - 'debian': { - $release_name = $::operatingsystemrelease ? { - # Debian releases - /^7\.[0-9]*$/ => 'wheezy', - /^8\.[0-9]*$/ => 'jessie', - /^9\.[0-9]*$/ => 'stretch', - /^10\.[0-9]*$/ => 'buster', - # Ubuntu releases - '12.04' => 'precise', - '14.04' => 'trusty', - '15.04' => 'vivid', - '15.10' => 'wily', - '16.04' => 'xenial', - '18.04' => 'bionic', - '20.04' => 'focal', - } - $base_packages = [ - # Accurate time is essential - 'ntp', - # Used in scripts including install-yarn.sh - 'curl', - 'wget', - # Used to read /etc/zulip/zulip.conf for `zulipconf` Puppet function - 'crudini', - # Used for tools like sponge - 'moreutils', - # Nagios monitoring plugins - $zulip::common::nagios_plugins, - # Required for using HTTPS in apt repositories. - 'apt-transport-https', - # Needed for the cron jobs installed by Puppet - 'cron', - ] - } - 'redhat': { - $release_name = "${::operatingsystem}${::operatingsystemmajrelease}" - $base_packages = [ - 'ntp', - 'curl', - 'wget', - 'crudini', - 'moreutils', - 'nmap-ncat', - 'nagios-plugins', # there is no dummy package on CentOS 7 - 'cronie', - ] - } - default: { - fail('osfamily not supported') - } - } - package { $base_packages: ensure => 'installed' } - - group { 'zulip': - ensure => present, - } - - user { 'zulip': - ensure => present, - require => Group['zulip'], - gid => 'zulip', - shell => '/bin/bash', - home => '/home/zulip', - managehome => true, - } - - file { '/etc/zulip': - ensure => 'directory', - mode => '0644', - owner => 'zulip', - group => 'zulip', - links => 'follow', - } - file { ['/etc/zulip/zulip.conf', '/etc/zulip/settings.py']: - ensure => 'file', - require => File['/etc/zulip'], - mode => '0644', - owner => 'zulip', - group => 'zulip', - } - file { '/etc/zulip/zulip-secrets.conf': - ensure => 'file', - require => File['/etc/zulip'], - mode => '0640', - owner => 'zulip', - group => 'zulip', - } - - file { '/etc/security/limits.conf': - ensure => file, - mode => '0640', - owner => 'root', - group => 'root', - source => 'puppet:///modules/zulip/limits.conf', - } - - # This directory is written to by cron jobs for reading by Nagios - file { '/var/lib/nagios_state/': - ensure => directory, - group => 'zulip', - mode => '0774', - } - - file { '/var/log/zulip': - ensure => 'directory', - owner => 'zulip', - group => 'zulip', - mode => '0640', - } - - file { "${zulip::common::nagios_plugins_dir}/zulip_base": - require => Package[$zulip::common::nagios_plugins], - recurse => true, - purge => true, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/zulip/nagios_plugins/zulip_base', - } + include zulip::profile::base } diff --git a/puppet/zulip/manifests/dockervoyager.pp b/puppet/zulip/manifests/dockervoyager.pp index 43f6c2afb0..d095acb7d0 100644 --- a/puppet/zulip/manifests/dockervoyager.pp +++ b/puppet/zulip/manifests/dockervoyager.pp @@ -1,27 +1,4 @@ -# This class includes all the modules you need to install/run a Zulip installation -# in a single container (without the database, memcached, Redis services). -# The database, memcached, Redis services need to be run in separate containers. -# Through this split of services, it is easier to scale the services to the needs. +# @summary Temporary shim for docker all-in-one profile class zulip::dockervoyager { - include zulip::base - include zulip::app_frontend - include zulip::supervisor - include zulip::process_fts_updates - - file { "${zulip::common::supervisor_conf_dir}/cron.conf": - ensure => file, - require => Package[supervisor], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/supervisor/conf.d/cron.conf', - } - file { "${zulip::common::supervisor_conf_dir}/nginx.conf": - ensure => file, - require => Package[supervisor], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/supervisor/conf.d/nginx.conf', - } + include zulip::profile::dockervoyager } diff --git a/puppet/zulip/manifests/memcached.pp b/puppet/zulip/manifests/memcached.pp index c6884f703b..37dc868e7d 100644 --- a/puppet/zulip/manifests/memcached.pp +++ b/puppet/zulip/manifests/memcached.pp @@ -1,107 +1,4 @@ +# @summary Temporary shim for memcached profile class zulip::memcached { - include zulip::base - include zulip::sasl_modules - include zulip::systemd_daemon_reload - - case $::osfamily { - 'debian': { - $memcached_packages = [ 'memcached', 'sasl2-bin' ] - $memcached_user = 'memcache' - } - 'redhat': { - $memcached_packages = [ 'memcached', 'cyrus-sasl' ] - $memcached_user = 'memcached' - } - default: { - fail('osfamily not supported') - } - } - package { $memcached_packages: ensure => 'installed' } - - $memcached_memory = zulipconf('memcached', 'memory', $zulip::common::total_memory_mb / 8) - file { '/etc/sasl2': - ensure => directory, - } - file { '/etc/sasl2/memcached-zulip-password': - # We cache the password in this file so we can check whether it - # changed and avoid running saslpasswd2 if it didn't. - require => File['/etc/sasl2'], - owner => 'root', - group => 'root', - mode => '0600', - content => zulipsecret('secrets', 'memcached_password', ''), - notify => Exec[generate_memcached_sasldb2], - } - file { '/var/lib/zulip/memcached-sasldb2.stamp': - owner => 'root', - group => 'root', - mode => '0644', - content => '1', - notify => Exec[generate_memcached_sasldb2], - } - exec { 'generate_memcached_sasldb2': - require => [ - Package[$memcached_packages], - Package[$zulip::sasl_modules::sasl_module_packages], - ], - refreshonly => true, - # Use localhost for the currently recommended MEMCACHED_USERNAME = - # "zulip@localhost" and the hostname for compatibility with - # MEMCACHED_USERNAME = "zulip". - command => "bash -euc ' -rm -f /etc/sasl2/memcached-sasldb2 -saslpasswd2 -p -f /etc/sasl2/memcached-sasldb2 \ - -a memcached -u localhost zulip < /etc/sasl2/memcached-zulip-password -saslpasswd2 -p -f /etc/sasl2/memcached-sasldb2 \ - -a memcached -u \"\$HOSTNAME\" zulip < /etc/sasl2/memcached-zulip-password -'", - } - file { '/etc/sasl2/memcached-sasldb2': - require => Exec[generate_memcached_sasldb2], - owner => $memcached_user, - group => $memcached_user, - mode => '0600', - } - file { '/etc/sasl2/memcached.conf': - require => File['/etc/sasl2'], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/sasl2/memcached.conf', - notify => Service[memcached], - } - file { '/etc/systemd/system/memcached.service.d': - ensure => directory, - } - file { '/etc/systemd/system/memcached.service.d/zulip-fix-sasl.conf': - require => File['/etc/systemd/system/memcached.service.d'], - owner => 'root', - group => 'root', - mode => '0644', - content => "\ -# https://bugs.launchpad.net/ubuntu/+source/memcached/+bug/1878721 -[Service] -Environment=SASL_CONF_PATH=/etc/sasl2 -", - notify => [ - Class['zulip::systemd_daemon_reload'], - Service['memcached'], - ], - } - file { '/etc/memcached.conf': - ensure => file, - require => [ - Package[$memcached_packages], - Package[$zulip::sasl_modules::sasl_module_packages] - ], - owner => 'root', - group => 'root', - mode => '0644', - content => template('zulip/memcached.conf.template.erb'), - } - service { 'memcached': - ensure => running, - subscribe => File['/etc/memcached.conf'], - require => Class['zulip::systemd_daemon_reload']; - } + include zulip::profile::memcached } diff --git a/puppet/zulip/manifests/nagios.pp b/puppet/zulip/manifests/nagios.pp index 044c70657b..f8a8fb9cfa 100644 --- a/puppet/zulip/manifests/nagios.pp +++ b/puppet/zulip/manifests/nagios.pp @@ -1,7 +1,7 @@ # This manifest installs Zulip's Nagios plugins intended to be on # localhost on a Nagios server. # -# Depends on zulip::base to have installed `monitoring-plugins-basic`. +# Depends on zulip::profile::base to have installed `monitoring-plugins-basic`. class zulip::nagios { file { "${zulip::common::nagios_plugins_dir}/zulip_nagios_server": require => Package[$zulip::common::nagios_plugins], diff --git a/puppet/zulip/manifests/postgres_appdb_tuned.pp b/puppet/zulip/manifests/postgres_appdb_tuned.pp index 6c2735fb6e..6d6348f8b0 100644 --- a/puppet/zulip/manifests/postgres_appdb_tuned.pp +++ b/puppet/zulip/manifests/postgres_appdb_tuned.pp @@ -1,42 +1,4 @@ -# postgres_appdb_tuned extends postgres_appdb_base by automatically -# generating tuned database configuration. +# @summary Temporary shim for postgres database server profile class zulip::postgres_appdb_tuned { - include zulip::base - include zulip::postgres_appdb_base - - $work_mem = $zulip::common::total_memory_mb / 512 - $shared_buffers = $zulip::common::total_memory_mb / 8 - $effective_cache_size = $zulip::common::total_memory_mb * 10 / 32 - $maintenance_work_mem = $zulip::common::total_memory_mb / 32 - - $random_page_cost = zulipconf('postgresql', 'random_page_cost', undef) - $effective_io_concurrency = zulipconf('postgresql', 'effective_io_concurrency', undef) - $replication = zulipconf('postgresql', 'replication', undef) - $listen_addresses = zulipconf('postgresql', 'listen_addresses', undef) - - $ssl_cert_file = zulipconf('postgresql', 'ssl_cert_file', undef) - $ssl_key_file = zulipconf('postgresql', 'ssl_key_file', undef) - $ssl_ca_file = zulipconf('postgresql', 'ssl_ca_file', undef) - - file { $zulip::postgres_appdb_base::postgres_confdirs: - ensure => directory, - owner => 'postgres', - group => 'postgres', - } - - $postgres_conf_file = "${zulip::postgres_appdb_base::postgres_confdir}/postgresql.conf" - file { $postgres_conf_file: - ensure => file, - require => Package[$zulip::postgres_appdb_base::postgresql], - owner => 'postgres', - group => 'postgres', - mode => '0644', - content => template("zulip/postgresql/${zulip::postgres_common::version}/postgresql.conf.template.erb"), - } - - exec { $zulip::postgres_appdb_base::postgres_restart: - require => Package[$zulip::postgres_appdb_base::postgresql], - refreshonly => true, - subscribe => [ File[$postgres_conf_file] ], - } + include zulip::profile::postgres_appdb_tuned } diff --git a/puppet/zulip/manifests/profile/app_frontend.pp b/puppet/zulip/manifests/profile/app_frontend.pp new file mode 100644 index 0000000000..ddd9a4b989 --- /dev/null +++ b/puppet/zulip/manifests/profile/app_frontend.pp @@ -0,0 +1,58 @@ +# Default configuration for a Zulip app frontend +class zulip::profile::app_frontend { + include zulip::profile::base + include zulip::app_frontend_base + include zulip::app_frontend_once + + $nginx_http_only = zulipconf('application_server', 'http_only', undef) + if $nginx_http_only != '' { + $nginx_listen_port = zulipconf('application_server', 'nginx_listen_port', 80) + } else { + $nginx_listen_port = zulipconf('application_server', 'nginx_listen_port', 443) + } + $no_serve_uploads = zulipconf('application_server', 'no_serve_uploads', undef) + $ssl_dir = $::osfamily ? { + 'debian' => '/etc/ssl', + 'redhat' => '/etc/pki/tls', + } + file { '/etc/nginx/sites-available/zulip-enterprise': + ensure => file, + require => Package[$zulip::common::nginx], + owner => 'root', + group => 'root', + mode => '0644', + content => template('zulip/nginx/zulip-enterprise.template.erb'), + notify => Service['nginx'], + } + file { '/etc/logrotate.d/zulip': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/logrotate/zulip', + } + file { '/etc/nginx/sites-enabled/zulip-enterprise': + ensure => 'link', + require => Package[$zulip::common::nginx], + target => '/etc/nginx/sites-available/zulip-enterprise', + notify => Service['nginx'], + } + + # Trigger 2x a day certbot renew + file { '/etc/cron.d/certbot-renew': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/cron.d/certbot-renew', + } + + # Restart the server regularly to avoid potential memory leak problems. + file { '/etc/cron.d/restart-zulip': + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/cron.d/restart-zulip', + } +} diff --git a/puppet/zulip/manifests/profile/base.pp b/puppet/zulip/manifests/profile/base.pp new file mode 100644 index 0000000000..21c892e112 --- /dev/null +++ b/puppet/zulip/manifests/profile/base.pp @@ -0,0 +1,138 @@ +# @summary Included only by classes that can be deployed. +# +# This class should only be included by classes that are intended to +# be able to be deployed on their own host. +class zulip::profile::base { + include zulip::common + case $::osfamily { + 'debian': { + include zulip::apt_repository + } + 'redhat': { + include zulip::yum_repository + } + default: { + fail('osfamily not supported') + } + } + case $::osfamily { + 'debian': { + $release_name = $::operatingsystemrelease ? { + # Debian releases + /^7\.[0-9]*$/ => 'wheezy', + /^8\.[0-9]*$/ => 'jessie', + /^9\.[0-9]*$/ => 'stretch', + /^10\.[0-9]*$/ => 'buster', + # Ubuntu releases + '12.04' => 'precise', + '14.04' => 'trusty', + '15.04' => 'vivid', + '15.10' => 'wily', + '16.04' => 'xenial', + '18.04' => 'bionic', + '20.04' => 'focal', + } + $base_packages = [ + # Accurate time is essential + 'ntp', + # Used in scripts including install-yarn.sh + 'curl', + 'wget', + # Used to read /etc/zulip/zulip.conf for `zulipconf` Puppet function + 'crudini', + # Used for tools like sponge + 'moreutils', + # Nagios monitoring plugins + $zulip::common::nagios_plugins, + # Required for using HTTPS in apt repositories. + 'apt-transport-https', + # Needed for the cron jobs installed by Puppet + 'cron', + ] + } + 'redhat': { + $release_name = "${::operatingsystem}${::operatingsystemmajrelease}" + $base_packages = [ + 'ntp', + 'curl', + 'wget', + 'crudini', + 'moreutils', + 'nmap-ncat', + 'nagios-plugins', # there is no dummy package on CentOS 7 + 'cronie', + ] + } + default: { + fail('osfamily not supported') + } + } + package { $base_packages: ensure => 'installed' } + + group { 'zulip': + ensure => present, + } + + user { 'zulip': + ensure => present, + require => Group['zulip'], + gid => 'zulip', + shell => '/bin/bash', + home => '/home/zulip', + managehome => true, + } + + file { '/etc/zulip': + ensure => 'directory', + mode => '0644', + owner => 'zulip', + group => 'zulip', + links => 'follow', + } + file { ['/etc/zulip/zulip.conf', '/etc/zulip/settings.py']: + ensure => 'file', + require => File['/etc/zulip'], + mode => '0644', + owner => 'zulip', + group => 'zulip', + } + file { '/etc/zulip/zulip-secrets.conf': + ensure => 'file', + require => File['/etc/zulip'], + mode => '0640', + owner => 'zulip', + group => 'zulip', + } + + file { '/etc/security/limits.conf': + ensure => file, + mode => '0640', + owner => 'root', + group => 'root', + source => 'puppet:///modules/zulip/limits.conf', + } + + # This directory is written to by cron jobs for reading by Nagios + file { '/var/lib/nagios_state/': + ensure => directory, + group => 'zulip', + mode => '0774', + } + + file { '/var/log/zulip': + ensure => 'directory', + owner => 'zulip', + group => 'zulip', + mode => '0640', + } + + file { "${zulip::common::nagios_plugins_dir}/zulip_base": + require => Package[$zulip::common::nagios_plugins], + recurse => true, + purge => true, + owner => 'root', + group => 'root', + mode => '0755', + source => 'puppet:///modules/zulip/nagios_plugins/zulip_base', + } +} diff --git a/puppet/zulip/manifests/profile/dockervoyager.pp b/puppet/zulip/manifests/profile/dockervoyager.pp new file mode 100644 index 0000000000..0be712a17c --- /dev/null +++ b/puppet/zulip/manifests/profile/dockervoyager.pp @@ -0,0 +1,27 @@ +# This class includes all the modules you need to install/run a Zulip installation +# in a single container (without the database, memcached, Redis services). +# The database, memcached, Redis services need to be run in separate containers. +# Through this split of services, it is easier to scale the services to the needs. +class zulip::profile::dockervoyager { + include zulip::profile::base + include zulip::profile::app_frontend + include zulip::supervisor + include zulip::process_fts_updates + + file { "${zulip::common::supervisor_conf_dir}/cron.conf": + ensure => file, + require => Package[supervisor], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/supervisor/conf.d/cron.conf', + } + file { "${zulip::common::supervisor_conf_dir}/nginx.conf": + ensure => file, + require => Package[supervisor], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/supervisor/conf.d/nginx.conf', + } +} diff --git a/puppet/zulip/manifests/profile/memcached.pp b/puppet/zulip/manifests/profile/memcached.pp new file mode 100644 index 0000000000..ad3c5226f1 --- /dev/null +++ b/puppet/zulip/manifests/profile/memcached.pp @@ -0,0 +1,107 @@ +class zulip::profile::memcached { + include zulip::profile::base + include zulip::sasl_modules + include zulip::systemd_daemon_reload + + case $::osfamily { + 'debian': { + $memcached_packages = [ 'memcached', 'sasl2-bin' ] + $memcached_user = 'memcache' + } + 'redhat': { + $memcached_packages = [ 'memcached', 'cyrus-sasl' ] + $memcached_user = 'memcached' + } + default: { + fail('osfamily not supported') + } + } + package { $memcached_packages: ensure => 'installed' } + + $memcached_memory = zulipconf('memcached', 'memory', $zulip::common::total_memory_mb / 8) + file { '/etc/sasl2': + ensure => directory, + } + file { '/etc/sasl2/memcached-zulip-password': + # We cache the password in this file so we can check whether it + # changed and avoid running saslpasswd2 if it didn't. + require => File['/etc/sasl2'], + owner => 'root', + group => 'root', + mode => '0600', + content => zulipsecret('secrets', 'memcached_password', ''), + notify => Exec[generate_memcached_sasldb2], + } + file { '/var/lib/zulip/memcached-sasldb2.stamp': + owner => 'root', + group => 'root', + mode => '0644', + content => '1', + notify => Exec[generate_memcached_sasldb2], + } + exec { 'generate_memcached_sasldb2': + require => [ + Package[$memcached_packages], + Package[$zulip::sasl_modules::sasl_module_packages], + ], + refreshonly => true, + # Use localhost for the currently recommended MEMCACHED_USERNAME = + # "zulip@localhost" and the hostname for compatibility with + # MEMCACHED_USERNAME = "zulip". + command => "bash -euc ' +rm -f /etc/sasl2/memcached-sasldb2 +saslpasswd2 -p -f /etc/sasl2/memcached-sasldb2 \ + -a memcached -u localhost zulip < /etc/sasl2/memcached-zulip-password +saslpasswd2 -p -f /etc/sasl2/memcached-sasldb2 \ + -a memcached -u \"\$HOSTNAME\" zulip < /etc/sasl2/memcached-zulip-password +'", + } + file { '/etc/sasl2/memcached-sasldb2': + require => Exec[generate_memcached_sasldb2], + owner => $memcached_user, + group => $memcached_user, + mode => '0600', + } + file { '/etc/sasl2/memcached.conf': + require => File['/etc/sasl2'], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/sasl2/memcached.conf', + notify => Service[memcached], + } + file { '/etc/systemd/system/memcached.service.d': + ensure => directory, + } + file { '/etc/systemd/system/memcached.service.d/zulip-fix-sasl.conf': + require => File['/etc/systemd/system/memcached.service.d'], + owner => 'root', + group => 'root', + mode => '0644', + content => "\ +# https://bugs.launchpad.net/ubuntu/+source/memcached/+bug/1878721 +[Service] +Environment=SASL_CONF_PATH=/etc/sasl2 +", + notify => [ + Class['zulip::systemd_daemon_reload'], + Service['memcached'], + ], + } + file { '/etc/memcached.conf': + ensure => file, + require => [ + Package[$memcached_packages], + Package[$zulip::sasl_modules::sasl_module_packages] + ], + owner => 'root', + group => 'root', + mode => '0644', + content => template('zulip/memcached.conf.template.erb'), + } + service { 'memcached': + ensure => running, + subscribe => File['/etc/memcached.conf'], + require => Class['zulip::systemd_daemon_reload']; + } +} diff --git a/puppet/zulip/manifests/profile/postgres_appdb_tuned.pp b/puppet/zulip/manifests/profile/postgres_appdb_tuned.pp new file mode 100644 index 0000000000..07f4d8dc47 --- /dev/null +++ b/puppet/zulip/manifests/profile/postgres_appdb_tuned.pp @@ -0,0 +1,42 @@ +# postgres_appdb_tuned extends postgres_appdb_base by automatically +# generating tuned database configuration. +class zulip::profile::postgres_appdb_tuned { + include zulip::profile::base + include zulip::postgres_appdb_base + + $work_mem = $zulip::common::total_memory_mb / 512 + $shared_buffers = $zulip::common::total_memory_mb / 8 + $effective_cache_size = $zulip::common::total_memory_mb * 10 / 32 + $maintenance_work_mem = $zulip::common::total_memory_mb / 32 + + $random_page_cost = zulipconf('postgresql', 'random_page_cost', undef) + $effective_io_concurrency = zulipconf('postgresql', 'effective_io_concurrency', undef) + $replication = zulipconf('postgresql', 'replication', undef) + $listen_addresses = zulipconf('postgresql', 'listen_addresses', undef) + + $ssl_cert_file = zulipconf('postgresql', 'ssl_cert_file', undef) + $ssl_key_file = zulipconf('postgresql', 'ssl_key_file', undef) + $ssl_ca_file = zulipconf('postgresql', 'ssl_ca_file', undef) + + file { $zulip::postgres_appdb_base::postgres_confdirs: + ensure => directory, + owner => 'postgres', + group => 'postgres', + } + + $postgres_conf_file = "${zulip::postgres_appdb_base::postgres_confdir}/postgresql.conf" + file { $postgres_conf_file: + ensure => file, + require => Package[$zulip::postgres_appdb_base::postgresql], + owner => 'postgres', + group => 'postgres', + mode => '0644', + content => template("zulip/postgresql/${zulip::postgres_common::version}/postgresql.conf.template.erb"), + } + + exec { $zulip::postgres_appdb_base::postgres_restart: + require => Package[$zulip::postgres_appdb_base::postgresql], + refreshonly => true, + subscribe => [ File[$postgres_conf_file] ], + } +} diff --git a/puppet/zulip/manifests/profile/rabbit.pp b/puppet/zulip/manifests/profile/rabbit.pp new file mode 100644 index 0000000000..c0a0477ff3 --- /dev/null +++ b/puppet/zulip/manifests/profile/rabbit.pp @@ -0,0 +1,75 @@ +class zulip::profile::rabbit { + include zulip::profile::base + $erlang = $::osfamily ? { + 'debian' => 'erlang-base', + 'redhat' => 'erlang', + } + $rabbit_packages = [# Needed to run RabbitMQ + $erlang, + 'rabbitmq-server', + ] + package { $rabbit_packages: ensure => 'installed' } + + # Removed 2020-09 in version 4.0; these lines can be removed in + # Zulip version 5.0 and later. + file { ['/etc/cron.d/rabbitmq-queuesize', '/etc/cron.d/rabbitmq-numconsumers']: + ensure => absent, + } + + file { '/etc/default/rabbitmq-server': + ensure => file, + require => Package[rabbitmq-server], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/rabbitmq/rabbitmq-server', + } + + file { '/etc/rabbitmq/rabbitmq.config': + ensure => file, + require => Package[rabbitmq-server], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/rabbitmq/rabbitmq.config', + } + + $rabbitmq_nodename = zulipconf('rabbitmq', 'nodename', '') + if $rabbitmq_nodename != '' { + file { '/etc/rabbitmq': + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + + file { '/etc/rabbitmq/rabbitmq-env.conf': + ensure => file, + require => File['/etc/rabbitmq'], + before => [Package[rabbitmq-server], Service[rabbitmq-server]], + owner => 'root', + group => 'root', + mode => '0644', + content => template('zulip/rabbitmq-env.conf.template.erb'), + } + } + # epmd doesn't have an init script, so we just check if it is + # running, and if it isn't, start it. Even in case of a race, this + # won't leak epmd processes, because epmd checks if one is already + # running and exits if so. + exec { 'epmd': + command => 'epmd -daemon', + unless => 'pgrep -f epmd >/dev/null', + require => Package[$erlang], + path => '/usr/bin/:/bin/', + } + + service { 'rabbitmq-server': + ensure => running, + require => [Exec['epmd'], + File['/etc/rabbitmq/rabbitmq.config'], + File['/etc/default/rabbitmq-server']], + } + + # TODO: Should also call exactly once "configure-rabbitmq" +} diff --git a/puppet/zulip/manifests/profile/redis.pp b/puppet/zulip/manifests/profile/redis.pp new file mode 100644 index 0000000000..0ed67e4800 --- /dev/null +++ b/puppet/zulip/manifests/profile/redis.pp @@ -0,0 +1,68 @@ +class zulip::profile::redis { + include zulip::profile::base + case $::osfamily { + 'debian': { + $redis = 'redis-server' + $redis_dir = '/etc/redis' + } + 'redhat': { + $redis = 'redis' + $redis_dir = '/etc' + } + default: { + fail('osfamily not supported') + } + } + $redis_packages = [ # The server itself + $redis, + ] + + package { $redis_packages: ensure => 'installed' } + + $file = "${redis_dir}/redis.conf" + $zulip_redisconf = "${redis_dir}/zulip-redis.conf" + $line = "include ${zulip_redisconf}" + exec { 'redis': + unless => "/bin/grep -Fxqe '${line}' '${file}'", + path => '/bin', + command => "bash -c \"(/bin/echo; /bin/echo '# Include Zulip-specific configuration'; /bin/echo '${line}') >> '${file}'\"", + require => [Package[$redis], + File[$zulip_redisconf], + Exec['rediscleanup-zuli-redis']], + } + + # Fix the typo in the path to $zulip_redisconf introduced in + # 071e32985c1207f20043e1cf28f82300d9f23f31 without triggering a + # redis restart. + $legacy_wrong_filename = "${redis_dir}/zuli-redis.conf" + exec { 'rediscleanup-zuli-redis': + onlyif => "test -e ${legacy_wrong_filename}", + command => " + mv ${legacy_wrong_filename} ${zulip_redisconf} + perl -0777 -pe ' + if (m|^\\Q${line}\\E\$|m) { + s|^\\n?(:?# Include Zulip-specific configuration\\n)?include \\Q${legacy_wrong_filename}\\E\\n||m; + } else { + s|^include \\Q${legacy_wrong_filename}\\E\$|${line}|m; + } + ' -i /etc/redis/redis.conf + ", + provider => shell, + } + + $redis_password = zulipsecret('secrets', 'redis_password', '') + file { $zulip_redisconf: + ensure => file, + require => [Package[$redis], Exec['rediscleanup-zuli-redis']], + owner => 'redis', + group => 'redis', + mode => '0640', + content => template('zulip/zulip-redis.template.erb'), + } + + service { $redis: + ensure => running, + subscribe => [File[$zulip_redisconf], + Exec['redis']], + } +} diff --git a/puppet/zulip/manifests/profile/thumbor.pp b/puppet/zulip/manifests/profile/thumbor.pp new file mode 100644 index 0000000000..8598cb8696 --- /dev/null +++ b/puppet/zulip/manifests/profile/thumbor.pp @@ -0,0 +1,25 @@ +class zulip::profile::thumbor { + include zulip::profile::base + include zulip::nginx + include zulip::supervisor + + file { "${zulip::common::supervisor_conf_dir}/thumbor.conf": + ensure => file, + require => Package[supervisor], + owner => 'root', + group => 'root', + mode => '0644', + source => 'puppet:///modules/zulip/supervisor/conf.d/thumbor.conf', + notify => Service[$zulip::common::supervisor_service], + } + + file { '/etc/nginx/zulip-include/app.d/thumbor.conf': + ensure => file, + require => Package[$zulip::common::nginx], + owner => 'root', + group => 'root', + mode => '0644', + notify => Service['nginx'], + source => 'puppet:///modules/zulip/nginx/zulip-include-app.d/thumbor.conf', + } +} diff --git a/puppet/zulip/manifests/profile/voyager.pp b/puppet/zulip/manifests/profile/voyager.pp new file mode 100644 index 0000000000..f9497de0f6 --- /dev/null +++ b/puppet/zulip/manifests/profile/voyager.pp @@ -0,0 +1,22 @@ +# This class includes all the modules you need to run an entire Zulip +# installation on a single server. If desired, you can split up the +# different `zulip::profile::*` components of a Zulip installation on +# different servers by using the modules below on different machines +# (the module list is stored in `puppet_classes` in +# /etc/zulip/zulip.conf). See the corresponding configuration in +# /etc/zulip/settings.py for how to find the various services is also +# required to make this work. +class zulip::profile::voyager { + include zulip::profile::base + include zulip::profile::app_frontend + include zulip::profile::postgres_appdb_tuned + include zulip::profile::redis + include zulip::profile::memcached + include zulip::profile::rabbit + if $::osfamily == debian { + # camo is only required on Debian-based systems as part of + # our migration towards not including camo at all. + include zulip::localhost_camo + } + include zulip::static_asset_compiler +} diff --git a/puppet/zulip/manifests/rabbit.pp b/puppet/zulip/manifests/rabbit.pp index 1d0bdb5636..972f2574c5 100644 --- a/puppet/zulip/manifests/rabbit.pp +++ b/puppet/zulip/manifests/rabbit.pp @@ -1,75 +1,4 @@ +# @summary Temporary shim for puppet profile class zulip::rabbit { - include zulip::base - $erlang = $::osfamily ? { - 'debian' => 'erlang-base', - 'redhat' => 'erlang', - } - $rabbit_packages = [# Needed to run RabbitMQ - $erlang, - 'rabbitmq-server', - ] - package { $rabbit_packages: ensure => 'installed' } - - # Removed 2020-09 in version 4.0; these lines can be removed in - # Zulip version 5.0 and later. - file { ['/etc/cron.d/rabbitmq-queuesize', '/etc/cron.d/rabbitmq-numconsumers']: - ensure => absent, - } - - file { '/etc/default/rabbitmq-server': - ensure => file, - require => Package[rabbitmq-server], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/rabbitmq/rabbitmq-server', - } - - file { '/etc/rabbitmq/rabbitmq.config': - ensure => file, - require => Package[rabbitmq-server], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/rabbitmq/rabbitmq.config', - } - - $rabbitmq_nodename = zulipconf('rabbitmq', 'nodename', '') - if $rabbitmq_nodename != '' { - file { '/etc/rabbitmq': - ensure => 'directory', - owner => 'root', - group => 'root', - mode => '0755', - } - - file { '/etc/rabbitmq/rabbitmq-env.conf': - ensure => file, - require => File['/etc/rabbitmq'], - before => [Package[rabbitmq-server], Service[rabbitmq-server]], - owner => 'root', - group => 'root', - mode => '0644', - content => template('zulip/rabbitmq-env.conf.template.erb'), - } - } - # epmd doesn't have an init script, so we just check if it is - # running, and if it isn't, start it. Even in case of a race, this - # won't leak epmd processes, because epmd checks if one is already - # running and exits if so. - exec { 'epmd': - command => 'epmd -daemon', - unless => 'pgrep -f epmd >/dev/null', - require => Package[$erlang], - path => '/usr/bin/:/bin/', - } - - service { 'rabbitmq-server': - ensure => running, - require => [Exec['epmd'], - File['/etc/rabbitmq/rabbitmq.config'], - File['/etc/default/rabbitmq-server']], - } - - # TODO: Should also call exactly once "configure-rabbitmq" + include zulip::profile::rabbit } diff --git a/puppet/zulip/manifests/redis.pp b/puppet/zulip/manifests/redis.pp index b06304802d..c226007e04 100644 --- a/puppet/zulip/manifests/redis.pp +++ b/puppet/zulip/manifests/redis.pp @@ -1,68 +1,4 @@ +# @summary Temporary shim for redis profile class zulip::redis { - include zulip::base - case $::osfamily { - 'debian': { - $redis = 'redis-server' - $redis_dir = '/etc/redis' - } - 'redhat': { - $redis = 'redis' - $redis_dir = '/etc' - } - default: { - fail('osfamily not supported') - } - } - $redis_packages = [ # The server itself - $redis, - ] - - package { $redis_packages: ensure => 'installed' } - - $file = "${redis_dir}/redis.conf" - $zulip_redisconf = "${redis_dir}/zulip-redis.conf" - $line = "include ${zulip_redisconf}" - exec { 'redis': - unless => "/bin/grep -Fxqe '${line}' '${file}'", - path => '/bin', - command => "bash -c \"(/bin/echo; /bin/echo '# Include Zulip-specific configuration'; /bin/echo '${line}') >> '${file}'\"", - require => [Package[$redis], - File[$zulip_redisconf], - Exec['rediscleanup-zuli-redis']], - } - - # Fix the typo in the path to $zulip_redisconf introduced in - # 071e32985c1207f20043e1cf28f82300d9f23f31 without triggering a - # redis restart. - $legacy_wrong_filename = "${redis_dir}/zuli-redis.conf" - exec { 'rediscleanup-zuli-redis': - onlyif => "test -e ${legacy_wrong_filename}", - command => " - mv ${legacy_wrong_filename} ${zulip_redisconf} - perl -0777 -pe ' - if (m|^\\Q${line}\\E\$|m) { - s|^\\n?(:?# Include Zulip-specific configuration\\n)?include \\Q${legacy_wrong_filename}\\E\\n||m; - } else { - s|^include \\Q${legacy_wrong_filename}\\E\$|${line}|m; - } - ' -i /etc/redis/redis.conf - ", - provider => shell, - } - - $redis_password = zulipsecret('secrets', 'redis_password', '') - file { $zulip_redisconf: - ensure => file, - require => [Package[$redis], Exec['rediscleanup-zuli-redis']], - owner => 'redis', - group => 'redis', - mode => '0640', - content => template('zulip/zulip-redis.template.erb'), - } - - service { $redis: - ensure => running, - subscribe => [File[$zulip_redisconf], - Exec['redis']], - } + include zulip::profile::redis } diff --git a/puppet/zulip/manifests/thumbor.pp b/puppet/zulip/manifests/thumbor.pp index 144b11efb3..948489a24d 100644 --- a/puppet/zulip/manifests/thumbor.pp +++ b/puppet/zulip/manifests/thumbor.pp @@ -1,25 +1,4 @@ +# @summary Temporary shim for thumbor profile class zulip::thumbor { - include zulip::base - include zulip::nginx - include zulip::supervisor - - file { "${zulip::common::supervisor_conf_dir}/thumbor.conf": - ensure => file, - require => Package[supervisor], - owner => 'root', - group => 'root', - mode => '0644', - source => 'puppet:///modules/zulip/supervisor/conf.d/thumbor.conf', - notify => Service[$zulip::common::supervisor_service], - } - - file { '/etc/nginx/zulip-include/app.d/thumbor.conf': - ensure => file, - require => Package[$zulip::common::nginx], - owner => 'root', - group => 'root', - mode => '0644', - notify => Service['nginx'], - source => 'puppet:///modules/zulip/nginx/zulip-include-app.d/thumbor.conf', - } + include zulip::profile::thumbor } diff --git a/puppet/zulip/manifests/voyager.pp b/puppet/zulip/manifests/voyager.pp index de541af9c9..873e4131ad 100644 --- a/puppet/zulip/manifests/voyager.pp +++ b/puppet/zulip/manifests/voyager.pp @@ -1,24 +1,4 @@ -# This class includes all the modules you need to run an entire Zulip -# installation on a single server. If desired, you can split up the -# different components of a Zulip installation on different servers by -# using the modules below on different machines (the module list is -# stored in `puppet_classes` in /etc/zulip/zulip.conf). In general, -# every machine should have `zulip::base` included, but the various -# service modules can be arranged on different machines or the same -# machine as desired (corresponding configuration in -# /etc/zulip/settings.py for how to find the various services is also -# required to make this work). +# @summary Temporary shim for all-in-one profile class zulip::voyager { - include zulip::base - include zulip::app_frontend - include zulip::postgres_appdb_tuned - include zulip::memcached - include zulip::rabbit - include zulip::redis - if $::osfamily == debian { - # camo is only required on Debian-based systems as part of - # our migration towards not including camo at all. - include zulip::localhost_camo - } - include zulip::static_asset_compiler + include zulip::profile::voyager } diff --git a/puppet/zulip/templates/postgresql/10/postgresql.conf.centos.template.erb b/puppet/zulip/templates/postgresql/10/postgresql.conf.centos.template.erb index 838be24c85..adf335f198 100644 --- a/puppet/zulip/templates/postgresql/10/postgresql.conf.centos.template.erb +++ b/puppet/zulip/templates/postgresql/10/postgresql.conf.centos.template.erb @@ -676,10 +676,10 @@ vacuum_freeze_min_age = 1000000000 vacuum_freeze_table_age = 1800000000 # Performance settings -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip/templates/postgresql/10/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/10/postgresql.conf.template.erb index 855352180a..91d2bafbc2 100644 --- a/puppet/zulip/templates/postgresql/10/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/10/postgresql.conf.template.erb @@ -676,10 +676,10 @@ vacuum_freeze_min_age = 1000000000 vacuum_freeze_table_age = 1800000000 # Performance settings -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip/templates/postgresql/11/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/11/postgresql.conf.template.erb index 855352180a..91d2bafbc2 100644 --- a/puppet/zulip/templates/postgresql/11/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/11/postgresql.conf.template.erb @@ -676,10 +676,10 @@ vacuum_freeze_min_age = 1000000000 vacuum_freeze_table_age = 1800000000 # Performance settings -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip/templates/postgresql/12/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/12/postgresql.conf.template.erb index a78474986e..fb345f8f12 100644 --- a/puppet/zulip/templates/postgresql/12/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/12/postgresql.conf.template.erb @@ -770,10 +770,10 @@ vacuum_freeze_table_age = 1800000000 # Performance settings max_connections = 1000 -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb index f331f2ecef..ab0bf26a8c 100644 --- a/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/9.5/postgresql.conf.template.erb @@ -648,10 +648,10 @@ vacuum_freeze_min_age = 1000000000 vacuum_freeze_table_age = 1800000000 # Performance settings -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb b/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb index b94c4f785a..017b753d2e 100644 --- a/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb +++ b/puppet/zulip/templates/postgresql/9.6/postgresql.conf.template.erb @@ -661,10 +661,10 @@ vacuum_freeze_min_age = 1000000000 vacuum_freeze_table_age = 1800000000 # Performance settings -maintenance_work_mem = <%= scope["zulip::postgres_appdb_tuned::maintenance_work_mem"] %>MB -effective_cache_size = <%= scope["zulip::postgres_appdb_tuned::effective_cache_size"] %>MB -work_mem = <%= scope["zulip::postgres_appdb_tuned::work_mem"] %>MB -shared_buffers = <%= scope["zulip::postgres_appdb_tuned::shared_buffers"] %>MB +maintenance_work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::maintenance_work_mem"] %>MB +effective_cache_size = <%= scope["zulip::profile::postgres_appdb_tuned::effective_cache_size"] %>MB +work_mem = <%= scope["zulip::profile::postgres_appdb_tuned::work_mem"] %>MB +shared_buffers = <%= scope["zulip::profile::postgres_appdb_tuned::shared_buffers"] %>MB wal_buffers = 4MB checkpoint_completion_target = 0.7 <% if @random_page_cost != '' -%> diff --git a/puppet/zulip_ops/manifests/app_frontend.pp b/puppet/zulip_ops/manifests/app_frontend.pp index 77888729b7..f551a426dd 100644 --- a/puppet/zulip_ops/manifests/app_frontend.pp +++ b/puppet/zulip_ops/manifests/app_frontend.pp @@ -1,7 +1,7 @@ class zulip_ops::app_frontend { include zulip::app_frontend_base - include zulip::memcached - include zulip::rabbit + include zulip::profile::memcached + include zulip::profile::rabbit include zulip::postfix_localmail include zulip::static_asset_compiler include zulip_ops::app_frontend_monitoring diff --git a/puppet/zulip_ops/manifests/base.pp b/puppet/zulip_ops/manifests/base.pp index e5d523d9b1..5ff253cc9e 100644 --- a/puppet/zulip_ops/manifests/base.pp +++ b/puppet/zulip_ops/manifests/base.pp @@ -1,5 +1,5 @@ class zulip_ops::base { - include zulip::base + include zulip::profile::base include zulip_ops::munin_node $org_base_packages = [# Management for our systems diff --git a/puppet/zulip_ops/manifests/postgres_appdb.pp b/puppet/zulip_ops/manifests/postgres_appdb.pp index 03b84c602e..096fe8e656 100644 --- a/puppet/zulip_ops/manifests/postgres_appdb.pp +++ b/puppet/zulip_ops/manifests/postgres_appdb.pp @@ -1,6 +1,6 @@ class zulip_ops::postgres_appdb { include zulip_ops::base - include zulip::postgres_appdb_tuned + include zulip::profile::postgres_appdb_tuned include zulip::postgres_backups $common_packages = ['xfsprogs'] diff --git a/puppet/zulip_ops/manifests/redis.pp b/puppet/zulip_ops/manifests/redis.pp index ce7efe27d4..4edb81345e 100644 --- a/puppet/zulip_ops/manifests/redis.pp +++ b/puppet/zulip_ops/manifests/redis.pp @@ -1,6 +1,6 @@ class zulip_ops::redis { include zulip_ops::base - include zulip::redis + include zulip::profile::redis # Need redis_password in its own file for Nagios file { '/var/lib/nagios/redis_password': diff --git a/scripts/lib/install b/scripts/lib/install index 5af3c5b2b8..5699debfa8 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -122,9 +122,10 @@ read -r -a APT_OPTIONS <<<"${APT_OPTIONS:-}" # Install additional packages. read -r -a ADDITIONAL_PACKAGES <<<"${ADDITIONAL_PACKAGES:-}" # Comma-separated list of Puppet manifests to install. default is -# zulip::voyager for an all-in-one system or zulip::dockervoyager for -# Docker. Use e.g. zulip::app_frontend for a Zulip frontend server. -PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::voyager}" +# zulip::profile::voyager for an all-in-one system or +# zulip::profile::dockervoyager for Docker. Use +# e.g. zulip::profile::app_frontend for a Zulip frontend server. +PUPPET_CLASSES="${PUPPET_CLASSES:-zulip::profile::voyager}" VIRTUALENV_NEEDED="${VIRTUALENV_NEEDED:-yes}" POSTGRES_VERSION="${POSTGRES_VERSION:-12}" @@ -232,7 +233,7 @@ EOF fi case ",$PUPPET_CLASSES," in - *,zulip::voyager,* | *,zulip::postgres_appdb_tuned,*) + *,zulip::profile::voyager,* | *,zulip::profile::postgres_appdb_tuned,*) if [ "$package_system" = apt ]; then # We're going to install Postgres from the Postgres apt # repository; this may conflict with the existing Postgres. @@ -286,7 +287,7 @@ elif [ "$package_system" = yum ]; then fi # Check early for missing SSL certificates -if [ "$PUPPET_CLASSES" = "zulip::voyager" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then +if [ "$PUPPET_CLASSES" = "zulip::profile::voyager" ] && [ -z "$USE_CERTBOT""$SELF_SIGNED_CERT" ] && { ! [ -e "/etc/ssl/private/zulip.key" ] || ! [ -e "/etc/ssl/certs/zulip.combined-chain.crt" ]; }; then set +x cat </dev/null; then set +x cat </dev/null; then set +x echo "FAILURE: Runnable queue processors declared in zerver/worker/queue_processors.py " - echo "do not match those in puppet/zulip/manifests/base.pp" + echo "do not match those in puppet/zulip/manifests/profile/base.pp" echo "See https://zulip.readthedocs.io/en/latest/subsystems/queuing.html for details." echo diff -ur /tmp/expected_queue_processors.txt /tmp/running_queue_processors.txt diff --git a/tools/setup/install-aws-server b/tools/setup/install-aws-server index c2a80414ab..7ec854f695 100755 --- a/tools/setup/install-aws-server +++ b/tools/setup/install-aws-server @@ -10,8 +10,8 @@ if [ -z "$SERVER" ] || [ -z "$ROLES" ]; then echo "Installs an empty Ubuntu server in AWS with a Zulip server role." echo echo " * server is the local part of the hostname (e.g. postgres0)" - echo " * roles is a list of Puppet rules to be passed to scripts/lib/install" - echo " E.g. 'zulip::base,zulip::postgres_common'" + echo " * roles is a comma-separated list of Puppet rules to be passed to scripts/lib/install" + echo " E.g. 'zulip::profile::postgres_appdb_tuned'" echo " * branch is used to override the default branch to install from." echo echo "Reads configuration from $HOME/.zulip-install-server.conf, which should look like:"