install: Support remote database services like RDS.

Documentation and variable names edited by tabbott.

docs/production/deployment.md

@@ -64,13 +64,88 @@ of managing chat.zulip.org and zulipchat.com.
 
 ### Using Zulip with Amazon RDS as the database
 
-You cannot use most third-party database-as-a-service provides like
+You can use DBaaS services like Amazon RDS for the Zulip database.
-Amazon RDS as the database provider with Zulip, because Zulip requires
+The experience is slightly degraded, in that most DBaaS provides don't
-one of two different [full-text search postgres
+include useful dictionary files in their installations and don't
-extensions](../subsystems/full-text-search.md) to power its search.
+provide a way to provide them yourself, resulting in a degraded
-Neither is available in Amazon RDS; there should be no issue with
+[full-text search](../subsystems/full-text-search.md) experience
-using Zulip with a different database-as-a-service provider as long as
+around issues dictionary files are relevant (e.g. stemming).
-one of those postgres extensions is available.
+
+You also need to pass some extra options to the Zulip installer in
+order to avoid it throwing an error when Zulip attempts to configure
+the database's dictionary files for full-text search; the details are
+below.
+
+#### Step 1: Setup Zulip
+
+Follow the [standard instructions](../production/install.md), with one
+change.  When running the installer, pass the `--remote-postgres`
+flag, e.g.:
+
+```
+sudo -s  # If not already root
+./zulip-server-*/scripts/setup/install --certbot \
+    --email=YOUR_EMAIL --hostname=YOUR_HOSTNAME \
+    --remote-postgres --postgres-missing-dictionaries
+```
+
+The script also installs and starts Postgres on the server by
+default. We don't need it, so run the following command to
+stop and disable the local Postgres server.
+
+```
+sudo service postgresql stop
+sudo update-rc.d postgresql disable
+```
+
+This complication will be removed in a future version.
+
+#### Step 2: Create the Postgres database
+
+Access an administrative `psql` shell on your postgres database, and
+run the commands in `scripts/setup/create-db.sql` to:
+
+* Create a database called `zulip`.
+* Create a user called `zulip`.
+* Now login with the `zulip` user to create a schema called
+  `zulip` in the `zulip` database. You might have to grant `create`
+  privileges first for the `zulip` user to do this.
+
+Depending on how authentication works for your postgres installation,
+you may also need to set a password for the Zulip user, generate a
+client certificate, or similar; consult the documentation for your
+database provider for the available options.
+
+#### Step 3: Configure Zulip to use the Postgres database
+
+In `/etc/zulip/settings.py` on your Zulip server, configure the
+following settings with details for how to connect to your postgres
+server.  Your database provider should provide these details.
+
+* `REMOTE_POSTGRES_HOST`: Name or IP address of the postgres server.
+* `REMOTE_POSTGRES_PORT`: Port on the postgres server.
+* `REMOTE_POSTGRES_SSLMODE`: SSL Mode used to connect to the server.
+
+If you're using password authentication, you should specify the
+password of the `zulip` user in /etc/zulip/zulip-secrets.conf as
+follows:
+
+```
+postgres_password = abcd1234
+```
+
+Now complete the installation by running the following command to ask
+the Zulip installer to initialize the postgres database. (Note: The
+options are different from before).
+
+```
+./zulip-server-*/scripts/setup/install --certbot \
+    --email=YOUR_EMAIL --hostname=YOUR_HOSTNAME \
+    --remote-postgres --postgres-missing-dictionaries
+
+# And then generate a realm creation link:
+su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
+```
 
 ## Using an alternate port
 

scripts/lib/install

@@ -13,6 +13,8 @@ Other options:
   --no-init-db
   --cacert
   --no-dist-upgrade
+  --postgres-missing-dictionaries
+  --remote-postgres
 
 The --hostname and --email options are required,
 unless --no-init-db is set and --certbot is not.
@@ -22,7 +24,7 @@ EOF
 
 # Shell option parsing.  Over time, we'll want to move some of the
 # environment variables below into this self-documenting system.
-args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,no-overwrite-settings,self-signed-cert,certbot,hostname:,email:,cacert: -n "$0" -- "$@")"
+args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,no-overwrite-settings,self-signed-cert,certbot,postgres-missing-dictionaries,remote-postgres,hostname:,email:,cacert: -n "$0" -- "$@")"
 eval "set -- $args"
 while true; do
     case "$1" in
@@ -35,6 +37,8 @@ while true; do
         --no-overwrite-settings) NO_OVERWRITE_SETTINGS=1; shift;;
         --no-init-db) NO_INIT_DB=1; shift;;
         --no-dist-upgrade) NO_DIST_UPGRADE=1; shift;;
+        --postgres-missing-dictionaries) POSTGRES_MISSING_DICTIONARIES=1; shift;;
+        --remote-postgres) REMOTE_POSTGRES=1; shift;;
         --) shift; break;;
     esac
 done
@@ -275,6 +279,14 @@ if [ "$DEPLOYMENT_TYPE" = "dockervoyager" ]; then
     has_postgres=1
 fi
 
+if [ -n "$POSTGRES_MISSING_DICTIONARIES" ]; then
+    export POSTGRES_MISSING_DICTIONARIES="true"
+fi
+
+if [ -n "$REMOTE_POSTGRES" ]; then
+  has_postgres=1
+fi
+
 # These server restarting bits should be moveable into puppet-land, ideally
 apt-get -y upgrade
 
@@ -364,15 +376,17 @@ if [ -e "/var/run/supervisor.sock" ]; then
     chown zulip:zulip /var/run/supervisor.sock
 fi
 
-if [ -n "$NO_INIT_DB" ]; then
+if [ -n "$NO_INIT_DB" ] || [ -n "$REMOTE_POSTGRES" ]; then
     set +x
     cat <<EOF
 
  Success!
 
- Stopping because --no-init-db was passed.  To complete the installation, run:
+ Stopping because --no-init-db or --remote-postgres was passed.
+ To complete the installation, configure postgres and then run:
 
    su zulip -c '/home/zulip/deployments/current/scripts/setup/initialize-database'
+   su zulip -c '/home/zulip/deployments/current/manage.py generate_realm_creation_link'
 EOF
     exit 0
 fi

zerver/migrations/0001_initial.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.2 on 2017-06-22 10:22
+import os
 
 import bitfield.models
 import django.contrib.auth.models
@@ -36,10 +37,47 @@ class Migration(migrations.Migration):
 
     initial = True
 
+    zulip_postgres_dictionaries_unavailable = os.getenv("POSTGRES_MISSING_DICTIONARIES", False)
     dependencies = [
         ('auth', '0001_initial'),
     ]
 
+    if zulip_postgres_dictionaries_unavailable:
+        fts_sql = """
+CREATE TEXT SEARCH CONFIGURATION zulip.english_us_search (COPY=pg_catalog.english);
+"""
+    else:
+        fts_sql = """
+CREATE TEXT SEARCH DICTIONARY english_us_hunspell
+  (template = ispell, DictFile = en_us, AffFile = en_us, StopWords = zulip_english);
+CREATE TEXT SEARCH CONFIGURATION zulip.english_us_search (COPY=pg_catalog.english);
+ALTER TEXT SEARCH CONFIGURATION zulip.english_us_search
+  ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part
+  WITH english_us_hunspell, english_stem;
+"""
+
+    fts_sql += """
+
+CREATE FUNCTION escape_html(text) RETURNS text IMMUTABLE LANGUAGE 'sql' AS $$
+  SELECT replace(replace(replace(replace(replace($1, '&', '&amp;'), '<', '&lt;'),
+                                 '>', '&gt;'), '"', '&quot;'), '''', '&#39;');
+$$ ;
+
+ALTER TABLE zerver_message ADD COLUMN search_tsvector tsvector;
+CREATE INDEX zerver_message_search_tsvector ON zerver_message USING gin(search_tsvector);
+ALTER INDEX zerver_message_search_tsvector SET (fastupdate = OFF);
+
+CREATE TABLE fts_update_log (id SERIAL PRIMARY KEY, message_id INTEGER NOT NULL);
+CREATE FUNCTION do_notify_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS
+  $$ BEGIN NOTIFY fts_update_log; RETURN NEW; END $$;
+CREATE TRIGGER fts_update_log_notify AFTER INSERT ON fts_update_log
+  FOR EACH STATEMENT EXECUTE PROCEDURE do_notify_fts_update_log();
+CREATE FUNCTION append_to_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS
+  $$ BEGIN INSERT INTO fts_update_log (message_id) VALUES (NEW.id); RETURN NEW; END $$;
+CREATE TRIGGER zerver_message_update_search_tsvector_async
+  BEFORE INSERT OR UPDATE OF subject, rendered_content ON zerver_message
+  FOR EACH ROW EXECUTE PROCEDURE append_to_fts_update_log();
+"""
     operations = [
         migrations.CreateModel(
             name='UserProfile',
@@ -388,34 +426,7 @@ class Migration(migrations.Migration):
             field=models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions'),
         ),
         migrations.RunSQL(
-            sql="""
+            sql=fts_sql,
-CREATE TEXT SEARCH DICTIONARY english_us_hunspell
-  (template = ispell, DictFile = en_us, AffFile = en_us, StopWords = zulip_english);
-CREATE TEXT SEARCH CONFIGURATION zulip.english_us_search (COPY=pg_catalog.english);
-ALTER TEXT SEARCH CONFIGURATION zulip.english_us_search
-  ALTER MAPPING FOR asciiword, asciihword, hword_asciipart, word, hword, hword_part
-  WITH english_us_hunspell, english_stem;
-
-CREATE FUNCTION escape_html(text) RETURNS text IMMUTABLE LANGUAGE 'sql' AS $$
-  SELECT replace(replace(replace(replace(replace($1, '&', '&amp;'), '<', '&lt;'),
-                                 '>', '&gt;'), '"', '&quot;'), '''', '&#39;');
-$$ ;
-
-ALTER TABLE zerver_message ADD COLUMN search_tsvector tsvector;
-CREATE INDEX zerver_message_search_tsvector ON zerver_message USING gin(search_tsvector);
-ALTER INDEX zerver_message_search_tsvector SET (fastupdate = OFF);
-
-CREATE TABLE fts_update_log (id SERIAL PRIMARY KEY, message_id INTEGER NOT NULL);
-CREATE FUNCTION do_notify_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS
-  $$ BEGIN NOTIFY fts_update_log; RETURN NEW; END $$;
-CREATE TRIGGER fts_update_log_notify AFTER INSERT ON fts_update_log
-  FOR EACH STATEMENT EXECUTE PROCEDURE do_notify_fts_update_log();
-CREATE FUNCTION append_to_fts_update_log() RETURNS trigger LANGUAGE plpgsql AS
-  $$ BEGIN INSERT INTO fts_update_log (message_id) VALUES (NEW.id); RETURN NEW; END $$;
-CREATE TRIGGER zerver_message_update_search_tsvector_async
-  BEFORE INSERT OR UPDATE OF subject, rendered_content ON zerver_message
-  FOR EACH ROW EXECUTE PROCEDURE append_to_fts_update_log();
-""",
         ),
         migrations.AlterModelManagers(
             name='userprofile',