From 6c97a36355b1bc83c01918ea8fba8ad0e2d80061 Mon Sep 17 00:00:00 2001 From: Vishnu KS Date: Thu, 12 Dec 2019 15:20:04 +0530 Subject: [PATCH] install: Support remote database services like RDS. Documentation and variable names edited by tabbott. --- docs/production/deployment.md | 89 ++++++++++++++++++++++++++++--- scripts/lib/install | 20 +++++-- zerver/migrations/0001_initial.py | 67 +++++++++++++---------- 3 files changed, 138 insertions(+), 38 deletions(-) diff --git a/docs/production/deployment.md b/docs/production/deployment.md index 1237382d8a..a8cc0a3765 100644 --- a/docs/production/deployment.md +++ b/docs/production/deployment.md @@ -64,13 +64,88 @@ of managing chat.zulip.org and zulipchat.com. ### Using Zulip with Amazon RDS as the database -You cannot use most third-party database-as-a-service provides like -Amazon RDS as the database provider with Zulip, because Zulip requires -one of two different [full-text search postgres -extensions](../subsystems/full-text-search.md) to power its search. -Neither is available in Amazon RDS; there should be no issue with -using Zulip with a different database-as-a-service provider as long as -one of those postgres extensions is available. +You can use DBaaS services like Amazon RDS for the Zulip database. +The experience is slightly degraded, in that most DBaaS provides don't +include useful dictionary files in their installations and don't +provide a way to provide them yourself, resulting in a degraded +[full-text search](../subsystems/full-text-search.md) experience +around issues dictionary files are relevant (e.g. stemming). + +You also need to pass some extra options to the Zulip installer in +order to avoid it throwing an error when Zulip attempts to configure +the database's dictionary files for full-text search; the details are +below. + +#### Step 1: Setup Zulip + +Follow the [standard instructions](../production/install.md), with one +change. When running the installer, pass the `--remote-postgres` +flag, e.g.: + +``` +sudo -s # If not already root +./zulip-server-*/scripts/setup/install --certbot \ + --email=YOUR_EMAIL --hostname=YOUR_HOSTNAME \ + --remote-postgres --postgres-missing-dictionaries +``` + +The script also installs and starts Postgres on the server by +default. We don't need it, so run the following command to +stop and disable the local Postgres server. + +``` +sudo service postgresql stop +sudo update-rc.d postgresql disable +``` + +This complication will be removed in a future version. + +#### Step 2: Create the Postgres database + +Access an administrative `psql` shell on your postgres database, and +run the commands in `scripts/setup/create-db.sql` to: + +* Create a database called `zulip`. +* Create a user called `zulip`. +* Now login with the `zulip` user to create a schema called + `zulip` in the `zulip` database. You might have to grant `create` + privileges first for the `zulip` user to do this. + +Depending on how authentication works for your postgres installation, +you may also need to set a password for the Zulip user, generate a +client certificate, or similar; consult the documentation for your +database provider for the available options. + +#### Step 3: Configure Zulip to use the Postgres database + +In `/etc/zulip/settings.py` on your Zulip server, configure the +following settings with details for how to connect to your postgres +server. Your database provider should provide these details. + +* `REMOTE_POSTGRES_HOST`: Name or IP address of the postgres server. +* `REMOTE_POSTGRES_PORT`: Port on the postgres server. +* `REMOTE_POSTGRES_SSLMODE`: SSL Mode used to connect to the server. + +If you're using password authentication, you should specify the +password of the `zulip` user in /etc/zulip/zulip-secrets.conf as +follows: + +``` +postgres_password = abcd1234 +``` + +Now complete the installation by running the following command to ask +the Zulip installer to initialize the postgres database. (Note: The +options are different from before). + +``` +./zulip-server-*/scripts/setup/install --certbot \ + --email=YOUR_EMAIL --hostname=YOUR_HOSTNAME \ + --remote-postgres --postgres-missing-dictionaries + +# And then generate a realm creation link: +su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link' +``` ## Using an alternate port diff --git a/scripts/lib/install b/scripts/lib/install index fb7f31092d..b21b2b13da 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -13,6 +13,8 @@ Other options: --no-init-db --cacert --no-dist-upgrade + --postgres-missing-dictionaries + --remote-postgres The --hostname and --email options are required, unless --no-init-db is set and --certbot is not. @@ -22,7 +24,7 @@ EOF # Shell option parsing. Over time, we'll want to move some of the # environment variables below into this self-documenting system. -args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,no-overwrite-settings,self-signed-cert,certbot,hostname:,email:,cacert: -n "$0" -- "$@")" +args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,no-overwrite-settings,self-signed-cert,certbot,postgres-missing-dictionaries,remote-postgres,hostname:,email:,cacert: -n "$0" -- "$@")" eval "set -- $args" while true; do case "$1" in @@ -35,6 +37,8 @@ while true; do --no-overwrite-settings) NO_OVERWRITE_SETTINGS=1; shift;; --no-init-db) NO_INIT_DB=1; shift;; --no-dist-upgrade) NO_DIST_UPGRADE=1; shift;; + --postgres-missing-dictionaries) POSTGRES_MISSING_DICTIONARIES=1; shift;; + --remote-postgres) REMOTE_POSTGRES=1; shift;; --) shift; break;; esac done @@ -275,6 +279,14 @@ if [ "$DEPLOYMENT_TYPE" = "dockervoyager" ]; then has_postgres=1 fi +if [ -n "$POSTGRES_MISSING_DICTIONARIES" ]; then + export POSTGRES_MISSING_DICTIONARIES="true" +fi + +if [ -n "$REMOTE_POSTGRES" ]; then + has_postgres=1 +fi + # These server restarting bits should be moveable into puppet-land, ideally apt-get -y upgrade @@ -364,15 +376,17 @@ if [ -e "/var/run/supervisor.sock" ]; then chown zulip:zulip /var/run/supervisor.sock fi -if [ -n "$NO_INIT_DB" ]; then +if [ -n "$NO_INIT_DB" ] || [ -n "$REMOTE_POSTGRES" ]; then set +x cat <', '>'), '"', '"'), '''', '''); +$$ ; + +ALTER TABLE zerver_message ADD COLUMN search_tsvector tsvector; +CREATE INDEX zerver_message_search_tsvector ON zerver_message USING gin(search_tsvector); +ALTER INDEX zerver_message_search_tsvector SET (fastupdate = OFF); + +CREATE TABLE fts_update_log (id SERIAL PRIMARY KEY, message_id INTEGER NOT NULL); +CREATE FUNCTION do_notify_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS + $$ BEGIN NOTIFY fts_update_log; RETURN NEW; END $$; +CREATE TRIGGER fts_update_log_notify AFTER INSERT ON fts_update_log + FOR EACH STATEMENT EXECUTE PROCEDURE do_notify_fts_update_log(); +CREATE FUNCTION append_to_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS + $$ BEGIN INSERT INTO fts_update_log (message_id) VALUES (NEW.id); RETURN NEW; END $$; +CREATE TRIGGER zerver_message_update_search_tsvector_async + BEFORE INSERT OR UPDATE OF subject, rendered_content ON zerver_message + FOR EACH ROW EXECUTE PROCEDURE append_to_fts_update_log(); +""" operations = [ migrations.CreateModel( name='UserProfile', @@ -388,34 +426,7 @@ class Migration(migrations.Migration): field=models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions'), ), migrations.RunSQL( - sql=""" -CREATE TEXT SEARCH DICTIONARY english_us_hunspell - (template = ispell, DictFile = en_us, AffFile = en_us, StopWords = zulip_english); -CREATE TEXT SEARCH CONFIGURATION zulip.english_us_search (COPY=pg_catalog.english); -ALTER TEXT SEARCH CONFIGURATION zulip.english_us_search - ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part - WITH english_us_hunspell, english_stem; - -CREATE FUNCTION escape_html(text) RETURNS text IMMUTABLE LANGUAGE 'sql' AS $$ - SELECT replace(replace(replace(replace(replace($1, '&', '&'), '<', '<'), - '>', '>'), '"', '"'), '''', '''); -$$ ; - -ALTER TABLE zerver_message ADD COLUMN search_tsvector tsvector; -CREATE INDEX zerver_message_search_tsvector ON zerver_message USING gin(search_tsvector); -ALTER INDEX zerver_message_search_tsvector SET (fastupdate = OFF); - -CREATE TABLE fts_update_log (id SERIAL PRIMARY KEY, message_id INTEGER NOT NULL); -CREATE FUNCTION do_notify_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS - $$ BEGIN NOTIFY fts_update_log; RETURN NEW; END $$; -CREATE TRIGGER fts_update_log_notify AFTER INSERT ON fts_update_log - FOR EACH STATEMENT EXECUTE PROCEDURE do_notify_fts_update_log(); -CREATE FUNCTION append_to_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS - $$ BEGIN INSERT INTO fts_update_log (message_id) VALUES (NEW.id); RETURN NEW; END $$; -CREATE TRIGGER zerver_message_update_search_tsvector_async - BEFORE INSERT OR UPDATE OF subject, rendered_content ON zerver_message - FOR EACH ROW EXECUTE PROCEDURE append_to_fts_update_log(); -""", + sql=fts_sql, ), migrations.AlterModelManagers( name='userprofile',