search: Remove now unnecessary tsearch_extra dependency.

Now that we're implemented tsearch_extras in pure postgres, we no
longer need a custom extension.  This should help us considerably, as
it means we no longer need to ship custom apt packages at all.

Fixes #467.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

Dockerfile-postgresql

@@ -1,31 +1,13 @@
 # To build run `docker build -f Dockerfile-postgresql .` from the root of the
 # zulip repo.
 
-# Install build tools and build tsearch_extras for the current postgres
+# Currently the postgres images do not support automatic upgrading of
-# version. Currently the postgres images do not support automatic upgrading of
 # the on-disk data in volumes. So the base image can not currently be upgraded
 # without users needing a manual pgdump and restore.
-FROM postgres:10
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        postgresql-server-dev-$PG_MAJOR \
-        postgresql-server-dev-all \
-        git \
-        build-essential \
-        fakeroot \
-        devscripts
-RUN git clone https://github.com/zulip/tsearch_extras.git \
-    && cd tsearch_extras \
-    && echo $PG_MAJOR > debian/pgversions \
-    && pg_buildext updatecontrol \
-    && debuild -b -uc -us
 
-# Install tsearch_extras, hunspell, zulip stop words, and run zulip database
+# Install hunspell, zulip stop words, and run zulip database
 # init.
 FROM postgres:10
-ENV TSEARCH_EXTRAS_VERSION=0.4
-ENV TSEARCH_EXTRAS_DEB=postgresql-${PG_MAJOR}-tsearch-extras_${TSEARCH_EXTRAS_VERSION}_amd64.deb
-COPY --from=0 /${TSEARCH_EXTRAS_DEB} /tmp
 COPY puppet/zulip/files/postgresql/zulip_english.stop /usr/share/postgresql/$PG_MAJOR/tsearch_data/zulip_english.stop
 COPY scripts/setup/create-db.sql /docker-entrypoint-initdb.d/zulip-create-db.sql
 COPY scripts/setup/create-pgroonga.sql /docker-entrypoint-initdb.d/zulip-create-pgroonga.sql
@@ -36,8 +18,6 @@ RUN apt-key add /tmp/pgroonga-debian.asc \
     && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
        hunspell-en-us \
        postgresql-${PG_MAJOR}-pgroonga \
-    && DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/${TSEARCH_EXTRAS_DEB} \
-    && rm /tmp/${TSEARCH_EXTRAS_DEB} \
     && ln -sf /var/cache/postgresql/dicts/en_us.dict "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.dict" \
     && ln -sf /var/cache/postgresql/dicts/en_us.affix "/usr/share/postgresql/$PG_MAJOR/tsearch_data/en_us.affix" \
     && rm -rf /var/lib/apt/lists/*

docs/development/setup-advanced.md

@@ -101,12 +101,6 @@ git remote add -f upstream https://github.com/zulip/zulip.git
 doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \
     memcached libmemcached py-Pillow py-cryptography py-cffi
 
-# Get tsearch_extras and build it (using a modified version which
-# aliases int4 on OpenBSD):
-git clone https://github.com/blablacio/tsearch_extras
-cd tsearch_extras
-gmake && sudo gmake install
-
 # Point environment to custom include locations and use newer GCC
 # (needed for Node modules):
 export CFLAGS="-I/usr/local/include -I/usr/local/include/sasl"

docs/overview/architecture-overview.md

@@ -27,8 +27,7 @@ adapter](https://github.com/zulip/hubot-zulip); integrations with
 [Jenkins](https://github.com/zulip/zulip-jenkins-plugin),
 [Puppet](https://github.com/matthewbarr/puppet-zulip),
 [Redmine](https://github.com/zulip/zulip-redmine-plugin), and
-[Trello](https://github.com/zulip/trello-to-zulip); our [full-text
+[Trello](https://github.com/zulip/trello-to-zulip);
-search PostgreSQL extension](https://github.com/zulip/tsearch_extras);
 and [many more](https://github.com/zulip/).
 
 We use [Transifex](https://www.transifex.com/zulip/zulip/) to do

docs/subsystems/full-text-search.md

@@ -17,13 +17,6 @@ feature](http://www.postgresql.org/docs/current/static/textsearch.html),
 with a custom set of English stop words to improve the quality of the
 search results.
 
-We use a small extension,
-[tsearch_extras](https://github.com/zulip/tsearch_extras), for
-highlighting of the matching words.  There is [some discussion of
-removing this extension, at least as an
-option](https://github.com/zulip/zulip/issues/467), so that Zulip can
-be used with database-as-a-service platforms.
-
 In order to optimize the performance of delivering messages, the
 full-text search index is updated for newly sent messages in the
 background, after the message has been delivered.  This background

docs/testing/testing.md

@@ -102,19 +102,7 @@ something valuable to helping keep Zulip bug-free.
 
 ### Possible testing issues
 
-- When running the test suite, if you get an error like this:
+- When running the test suite, if you get an error involving Git that looks like this:
-
-  ```
-      sqlalchemy.exc.ProgrammingError: (ProgrammingError) function ts_match_locs_array(unknown, text, tsquery) does not   exist
-      LINE 2: ...ECT message_id, flags, subject, rendered_content, ts_match_l...
-                                                                   ^
-  ```
-
-  … then you need to install tsearch-extras, described
-  above. Afterwards, re-run the `init*-db` and the
-  `do-destroy-rebuild*-database` scripts.
-
-- Or, when running the test suite, if you get an error involving Git that looks like this:
 
   ```
       gitlint| An error occurred while executing '/usr/bin/git rev-list --max-count=-1 upstream/master..HEAD': b"fatal: ambiguous argument 'upstream/master..HEAD': unknown revision or path not in the working tree.\nUse '--' to separate paths from revisions, like this:\n'git <command> [<revision>...] -- [<file>...]'"

puppet/zulip/manifests/postgres_appdb_base.pp

@@ -8,24 +8,6 @@ class zulip::postgres_appdb_base {
     'debian': {
       include zulip::apt_repository
       $postgresql = "postgresql-${zulip::base::postgres_version}"
-      $appdb_packages = [
-        # Needed for our full text search system
-        "${postgresql}-tsearch-extras",
-      ]
-      if $zulip::base::release_name == 'stretch' {
-        zulip::safepackage {
-        $appdb_packages:
-          ensure  => 'installed',
-          require => Exec['setup_apt_repo'],
-        }
-      }
-      else {
-        exec {'build_tsearch_extras':
-          require => Package[$zulip::postgres_common::postgresql_dev],
-          command => "bash -c ${::zulip_scripts_path}/lib/build-tsearch-extras",
-          creates => "/usr/pgsql-${zulip::base::postgres_version}/lib/tsearch_extras.so",
-        }
-      }
       $postgres_sharedir = "/usr/share/postgresql/${zulip::base::postgres_version}"
       $tsearch_datadir = "${postgres_sharedir}/tsearch_data"
       $pgroonga_setup_sql_path = "${postgres_sharedir}/pgroonga_setup.sql"
@@ -34,10 +16,6 @@ class zulip::postgres_appdb_base {
     'redhat': {
       include zulip::yum_repository
       $postgresql = "postgresql${zulip::base::postgres_version}"
-      exec {'build_tsearch_extras':
-        command => "bash -c ${::zulip_scripts_path}/lib/build-tsearch-extras",
-        creates => "/usr/pgsql-${zulip::base::postgres_version}/lib/tsearch_extras.so",
-      }
       $postgres_sharedir = "/usr/pgsql-${zulip::base::postgres_version}/share"
       $tsearch_datadir = "${postgres_sharedir}/tsearch_data/"
       $pgroonga_setup_sql_path = "${postgres_sharedir}/pgroonga_setup.sql"

puppet/zulip/manifests/postgres_common.pp

@@ -3,12 +3,9 @@ class zulip::postgres_common {
   case $::osfamily {
     'debian': {
       $postgresql = "postgresql-${zulip::base::postgres_version}"
-      $postgresql_dev = "postgresql-server-dev-${zulip::base::postgres_version}"
       $postgres_packages = [
         # The database itself
         $postgresql,
-        # Developer pacakge for building extensions (such as tsearch_extras)
-        $postgresql_dev,
         # tools for database monitoring; formerly ptop
         'pgtop',
         # Needed just to support adding postgres user to 'zulip' group

scripts/lib/build-tsearch-extras

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-set -x
-set -e
-
-cd "$(mktemp -d)"
-distro=$(lsb_release -is)
-
-git clone https://github.com/zulip/tsearch_extras
-if [ "$distro" = "Ubuntu" ] || [ "$distro" == "Debian" ]; then
-    make="make"
-else
-    make="gmake"
-fi
-(cd tsearch_extras && "$make" && "$make" install)

scripts/setup/create-db.sql

@@ -3,4 +3,3 @@ ALTER ROLE zulip SET search_path TO zulip,public;
 CREATE DATABASE zulip OWNER=zulip;
 \connect zulip
 CREATE SCHEMA zulip AUTHORIZATION zulip;
-CREATE EXTENSION tsearch_extras SCHEMA zulip;

tools/lib/provision.py

@@ -171,32 +171,23 @@ COMMON_YUM_DEPENDENCIES = COMMON_DEPENDENCIES + [
     "libstdc++"
 ] + YUM_THUMBOR_VENV_DEPENDENCIES
 
-BUILD_TSEARCH_FROM_SOURCE = False
 BUILD_PGROONGA_FROM_SOURCE = False
 if vendor == "ubuntu" and os_version in ("18.10", "19.04"):
-    # For platforms without a tsearch-extras package distributed
-    # from our PPA, we need to build from source.
-    BUILD_TSEARCH_FROM_SOURCE = True
     SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql-{0}",
             "postgresql-{0}-pgroonga",
-            # Dependency for building tsearch_extras from source
-            "postgresql-server-dev-{0}",
         ]
     ]
 elif vendor == 'debian' and os_version == "10":
-    # For platforms without a tsearch-extras package distributed
+    # For platforms without a pgroonga release, we need to build it
-    # from our PPA or a pgroonga release, we need to build both
     # from source.
     BUILD_PGROONGA_FROM_SOURCE = True
-    BUILD_TSEARCH_FROM_SOURCE = True
     SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql-{0}",
-            # Dependency for building tsearch_extras from source
-            "postgresql-server-dev-{0}",
             # Dependency for building pgroonga from source
+            "postgresql-server-dev-{0}",
             "libgroonga-dev",
             "libmsgpack-dev",
         ]
@@ -206,7 +197,6 @@ elif vendor in ["ubuntu", "debian"]:
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql-{0}",
             "postgresql-{0}-pgroonga",
-            "postgresql-{0}-tsearch-extras",
         ]
     ]
 elif vendor in ["CentOS", "RedHat"]:
@@ -214,11 +204,9 @@ elif vendor in ["CentOS", "RedHat"]:
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql{0}-server",
             "postgresql{0}",
-            "postgresql{0}-devel",
             "postgresql{0}-pgroonga",
         ]
     ] + REDHAT_VENV_DEPENDENCIES
-    BUILD_TSEARCH_FROM_SOURCE = True
 elif vendor == "Fedora":
     SYSTEM_DEPENDENCIES = COMMON_YUM_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
@@ -230,7 +218,6 @@ elif vendor == "Fedora":
             "msgpack-devel",
         ]
     ] + FEDORA_VENV_DEPENDENCIES
-    BUILD_TSEARCH_FROM_SOURCE = True
     BUILD_PGROONGA_FROM_SOURCE = True
 
 if family == 'redhat':
@@ -259,12 +246,10 @@ def install_system_deps():
     else:
         raise AssertionError("Invalid vendor")
 
-    # For some platforms, there aren't published pgroonga or
+    # For some platforms, there aren't published pgroonga
-    # tsearch-extra packages available, so we build them from source.
+    # packages available, so we build them from source.
     if BUILD_PGROONGA_FROM_SOURCE:
         run_as_root(["./scripts/lib/build-pgroonga"])
-    if BUILD_TSEARCH_FROM_SOURCE:
-        run_as_root(["./scripts/lib/build-tsearch-extras"])
 
 def install_apt_deps(deps_to_install):
     # type: (List[str]) -> None

tools/setup/postgres-init-dev-db

@@ -98,7 +98,6 @@ CREATE SCHEMA zulip;
 EOF
 
 "${ROOT_POSTGRES[@]}" -v ON_ERROR_STOP=1 -e "$DBNAME_BASE" << EOF
-CREATE EXTENSION tsearch_extras SCHEMA zulip;
 CREATE EXTENSION pgroonga;
 GRANT USAGE ON SCHEMA pgroonga TO $USERNAME;
 EOF

version.py

@@ -26,4 +26,4 @@ LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-relea
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '51.0'
+PROVISION_VERSION = '52.0'