ci: Move backend and production tests to Ubuntu 16.04 (xenial).

This is preparation for dropping support for Trusty in CI.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

.circleci/config.yml

@@ -82,7 +82,7 @@ aliases:
          || echo "Error in uploading coverage reports to codecov.io."
 
 jobs:
-  "trusty-backend-frontend-python3.4":
+  "trusty-backend-python3.4":
     docker:
       # This is built from tools/circleci/images/trusty/Dockerfile .
       # Trusty ships with Python 3.4.
@@ -90,6 +90,25 @@ jobs:
 
     working_directory: ~/zulip
 
+    steps:
+      - checkout
+
+      - *create_cache_directories
+      - *restore_cache_package_json
+      - *restore_cache_requirements
+      - *install_dependencies
+      - *save_cache_package_json
+      - *save_cache_requirements
+      - *run_backend_tests
+
+  "xenial-backend-frontend-python3.5":
+    docker:
+      # This is built from tools/circleci/images/xenial/Dockerfile .
+      # Xenial ships with Python 3.5.
+      - image: gregprice/circleci:xenial-python-4.test
+
+    working_directory: ~/zulip
+
     steps:
       - checkout
 
@@ -110,24 +129,6 @@ jobs:
       #     # also /tmp/zulip-test-event-log/
       #     destination: test-reports
 
-  "xenial-backend-python3.5":
-    docker:
-      # This is built from tools/circleci/images/xenial/Dockerfile .
-      # Xenial ships with Python 3.5.
-      - image: gregprice/circleci:xenial-python-4.test
-
-    working_directory: ~/zulip
-
-    steps:
-      - checkout
-      - *create_cache_directories
-      - *restore_cache_package_json
-      - *restore_cache_requirements
-      - *install_dependencies
-      - *save_cache_package_json
-      - *save_cache_requirements
-      - *run_backend_tests
-
   "bionic-backend-python3.6":
     docker:
       # This is built from tools/circleci/images/bionic/Dockerfile .
@@ -159,6 +160,6 @@ workflows:
   version: 2
   build:
     jobs:
-      - "trusty-backend-frontend-python3.4"
+      - "trusty-backend-python3.4"
-      - "xenial-backend-python3.5"
+      - "xenial-backend-frontend-python3.5"
       - "bionic-backend-python3.6"

.travis.yml

@@ -1,7 +1,6 @@
 # See https://zulip.readthedocs.io/en/latest/testing/continuous-integration.html for
 # high-level documentation on our Travis CI setup.
-dist: trusty
+dist: xenial
-group: deprecated-2017Q4
 install:
   # Disable sometimes-broken sources.list in Travis base images
   - sudo rm -vf /etc/apt/sources.list.d/*
@@ -40,15 +39,15 @@ env:
   global:
     - BOTO_CONFIG=/nonexistent
 language: python
-# Our test suites generally run on Python 3.4, the version in
+# Our test suites generally run on Python 3.5, the version in
-# Ubuntu 14.04 trusty, which is the oldest OS release we support.
+# Ubuntu 16.04 xenial, which is the oldest OS release we support.
 matrix:
   include:
     # Travis will actually run the jobs in the order they're listed here;
     # that doesn't seem to be documented, but it's what we see empirically.
     # We only get 4 jobs running at a time, so we try to make the first few
     # the most likely to break.
-    - python: "3.4"
+    - python: "3.5"
       env: TEST_SUITE=production
     # Other suites moved to CircleCI -- see .circleci/.
 sudo: required
@@ -59,7 +58,7 @@ addons:
       # debugging test flakes.
       - $(ls var/casper/* | tr "\n" ":")
       - $(ls /tmp/zulip-test-event-log/* | tr "\n" ":")
-  postgresql: "9.3"
+  postgresql: "9.5"
   apt:
     packages:
       - moreutils

docs/testing/continuous-integration.md

@@ -60,7 +60,7 @@ We currently run several jobs during a CircleCI build. They are:
 Each runs the Zulip backend test suites, using the indicated
 platform/OS and Python version. `bionic-python-3.6` job for example
 runs the tests in Ubuntu Xenial with Python 3.6 pre-installed.
-Additionally, the `trusty` suite also runs the Zulip frontend test
+Additionally, the `xenial` suite also runs the Zulip frontend test
 suites; since those are not platform-dependent, it doesn't make sense
 to run them on all platforms.  Your build for the PR will pass only if
 all the 3 jobs are executed successfully.

scripts/lib/check-upstart

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 release=$(lsb_release -sc)
 
-if [ "$release" = "xenial" ] && [ -x /sbin/start ] && [ -x /sbin/stop ] && [ -x /sbin/restart ] && [ -x /sbin/status ] && [ -x /sbin/initctl ]; then
+if [ "$release" = "xenial" ] && [ -x /sbin/start ] && [ -x /sbin/stop ] && [ -x /sbin/restart ] && [ -x /sbin/status ] && [ -x /sbin/initctl ] && [ ! "$TRAVIS" ]; then
     echo "You appear to be running Ubuntu Xenial, but with the upstart package installed."
     echo "In Ubuntu Xenial, upstart has been replaced by systemd, and having upstart"
     echo "installed will break the init scripts for Zulip dependencies like nginx."

tools/ci/production-helper

@@ -14,12 +14,12 @@ apt-get update
 # Hold upgrades to packages which are expensive to upgrade due to size
 # or computational cost (e.g. initramfs rebuilds) and aren't really
 # used by Zulip in production.
-apt-mark hold initramfs-tools initramfs-tools-bin oracle-java8-installer oracle-java9-installer udev base-files linux-firmware chromium-browser google-chrome-stable g++-4.8 gcc-4.8 cpp-4.8 openjdk-7-jre-headless linux-image-generic-lts-xenial
+apt-mark hold initramfs-tools initramfs-tools-bin udev base-files linux-firmware chromium-browser google-chrome-stable g++-4.8 gcc-4.8 cpp-4.8 linux-image-generic-lts-xenial
 # And hold tons more packages that aren't expensive to upgrade but
 # there are a lot of.  This is super ugly, but since Travis CI's
 # machines never update, we can avoid years of package upgrades (takes
 # ~80s to install) by doing this.
-apt-mark hold accountsservice apparmor apport apt apt-transport-https apt-utils bash bash-completion bind9-host binutils binutils-doc bsdutils bzr cloud-guest-utils cloud-init coreutils cpio dbus dnsutils dosfstools dpkg dpkg-dev e2fslibs e2fsprogs eject gcc-4.9-base git-core grub-common grub-pc grub-pc-bin grub2-common icedtea-7-plugin icedtea-netx icedtea-netx-common icu-devtools ifupdown imagemagick imagemagick-common init-system-helpers initscripts irqbalance isc-dhcp-client isc-dhcp-common klibc-utils krb5-locales krb5-multidev landscape-client landscape-common libaccountsservice0 libapparmor-perl libapparmor1 libapt-inst1.5 libapt-pkg4.12 libarchive13 libbind9-90 libblkid1 libc-bin libc-dev-bin libc6 libc6-dev libcdt5 libcgmanager0 libcgraph6 libcups2 libcurl3-gnutls libdbus-1-3 libdns100 libdpkg-perl libdrm-dev libdrm-intel1 libdrm-nouveau2 libdrm-radeon1 libdrm2 libevent-2.0-5 libexpat1 libexpat1-dev libgc1c2 libgcc1 libgcrypt11 libgcrypt11-dev libgd3 libgl1-mesa-dev libgl1-mesa-dri libgl1-mesa-glx libglapi-mesa libgnutls-dev libgnutls-openssl27 libgnutls26 libgnutlsxx27 libgraphite2-3 libgraphviz-dev libgssapi-krb5-2 libgssrpc4 libgstreamer-plugins-base1.0-0 libgtk2.0-0 libgtk2.0-common libgvc6 libgvpr2 libicu-dev libicu52 libisc95 libisccc90 libisccfg90 libjasper-dev libjasper1 libk5crypto3 libkadm5clnt-mit9 libkadm5srv-mit9 libkdb5-7 libklibc libkrb5-3 libkrb5-dev libkrb5support0 liblcms2-2 liblcms2-dev liblwres90 libmagickcore-dev libmagickcore5 libmagickcore5-extra libmagickwand-dev libmagickwand5 libmount1 libmysqlclient-dev libmysqlclient18 libnettle4 libnl-3-200 libnl-genl-3-200 libnspr4 libnss3 libnss3-nssdb libnuma1 libpam-modules libpam-modules-bin libpam-runtime libpam-systemd libpam0g libpam0g-dev libpathplan4 libpci3 libpcre3 libpcre3-dev libpcrecpp0 libpcsclite1 libpixman-1-0 libpixman-1-dev libpng12-0 libpng12-dev libpolkit-agent-1-0 libpolkit-backend-1-0 libpolkit-gobject-1-0 libpython3.4 libpython3.4-dev libpython3.4-minimal libpython3.4-stdlib librtmp-dev librtmp0 libsndfile1 libspice-server1 libss2 libssl-dev libssl1.0.0 libsystemd-daemon0 libsystemd-journal0 libsystemd-login0 libtasn1-6 libtasn1-6-dev libtdb1 libtiff5 libtiff5-dev libtiffxx5 libuuid1 libxdot4 libxml2 libxml2-dev libxpm-dev libxpm4 linux-libc-dev login lsb-base lshw makedev mesa-common-dev mongodb-org mongodb-org-mongos mongodb-org-server mongodb-org-shell mongodb-org-tools mount multiarch-support mysql-common ntpdate openssh-client openssh-server openssh-sftp-server oracle-java9-set-default os-prober overlayroot passwd pciutils perl perl-base perl-modules pgdg-keyring policykit-1 pollinate postgresql-client postgresql-client-common postgresql-common python-apt python-apt-common python-bzrlib  python-urllib3 python3-apport python3-apt python3-distupgrade python3-gdbm python3-problem-report python3-software-properties python3-update-manager python3.4 python3.4-dev python3.4-minimal rsync scons software-properties-common sudo systemd-services sysv-rc sysvinit-utils tar tcpdump tzdata tzdata-java ubuntu-release-upgrader-core unattended-upgrades unzip update-manager-core usbutils util-linux uuid-runtime w3m xserver-xorg-video-intel
+apt-mark hold accountsservice apparmor apport apt apt-transport-https apt-utils bash bash-completion bind9-host binutils binutils-doc bsdutils bzr cloud-guest-utils cloud-init coreutils cpio dbus dnsutils dosfstools dpkg dpkg-dev e2fslibs e2fsprogs eject gcc-4.9-base git-core grub-common grub-pc grub-pc-bin grub2-common icu-devtools ifupdown imagemagick imagemagick-common init-system-helpers initscripts irqbalance isc-dhcp-client isc-dhcp-common klibc-utils krb5-locales krb5-multidev libaccountsservice0 libapparmor-perl libapparmor1 libblkid1 libc-bin libc-dev-bin libc6 libc6-dev libcdt5 libcgmanager0 libcgraph6 libcups2 libcurl3-gnutls libdbus-1-3 libdpkg-perl libdrm-intel1 libdrm-nouveau2 libdrm-radeon1 libdrm2 libevent-2.0-5 libexpat1 libexpat1-dev libgc1c2 libgcc1 libgd3 libgl1-mesa-dri libgl1-mesa-glx libglapi-mesa libgnutls-openssl27 libgraphite2-3 libgraphviz-dev libgssapi-krb5-2 libgssrpc4 libgtk2.0-0 libgtk2.0-common libgvc6 libgvpr2 libicu-dev libjasper-dev libjasper1 libk5crypto3 libkadm5clnt-mit9 libkadm5srv-mit9 libklibc libkrb5-3 libkrb5-dev libkrb5support0 liblcms2-2 liblcms2-dev libmagickwand-dev libmount1 libmysqlclient-dev libnl-3-200 libnl-genl-3-200 libnspr4 libnss3 libnss3-nssdb libnuma1 libpam-modules libpam-modules-bin libpam-runtime libpam-systemd libpam0g libpam0g-dev libpathplan4 libpci3 libpcre3 libpcre3-dev libpcsclite1 libpixman-1-0 libpixman-1-dev libpng12-0 libpng12-dev libpolkit-agent-1-0 libpolkit-backend-1-0 libpolkit-gobject-1-0 libpython3.4 libsndfile1 libss2 libssl-dev libssl1.0.0 libtasn1-6 libtiff5 libtiff5-dev libtiffxx5 libuuid1 libxdot4 libxml2 libxml2-dev libxpm4 linux-libc-dev login lsb-base lshw makedev mongodb-org mongodb-org-mongos mongodb-org-server mongodb-org-shell mongodb-org-tools mount multiarch-support mysql-client-5.7 mysql-client-core-5.7 mysql-common mysql-server-5.7 mysql-server-core-5.7 ntpdate openssh-client openssh-server openssh-sftp-server os-prober overlayroot passwd pciutils perl perl-base pgdg-keyring policykit-1 pollinate postgresql-client postgresql-client-common postgresql-common python-apt python-apt-common python-bzrlib  python-urllib3 python3-apport python3-apt python3-distupgrade python3-gdbm python3-problem-report python3-software-properties python3-update-manager python3.4 python3.4-minimal rsync software-properties-common sudo sysv-rc sysvinit-utils tar tcpdump tzdata ubuntu-release-upgrader-core unzip update-manager-core usbutils util-linux uuid-runtime w3m
 
 if ! apt-get dist-upgrade -y "${APT_OPTIONS[@]}"; then
     echo "\`apt-get dist-upgrade\`: Failure occured while trying to perform distribution upgrade, Retrying..."

tools/ci/setup-production

@@ -19,7 +19,9 @@ sudo chmod a+rX /home/travis
 # Uninstall the unnecessary extra versions of postgres that Travis CI
 # installs since if we don't do this, doing apt upgrades can fail due
 # to conflicts over which version of postgres should be running.
-sudo apt-get remove postgresql-9.2 postgresql-client-9.2 postgresql-contrib-9.2 postgresql-9.4 postgresql-client-9.4 postgresql-9.5  postgresql-contrib-9.5 postgresql-client-9.5 postgresql-9.6 postgresql-contrib-9.6 postgresql-client-9.6 -y
+sudo apt-get -y remove \
+    postgresql-9.4 postgresql-contrib-9.4 postgresql-client-9.4 \
+    postgresql-9.6 postgresql-contrib-9.6 postgresql-client-9.6
 
 # Remove some of Travis's' stupid extra sources.list files
 sudo rm -f /etc/apt/sources.list.d/mongodb*.list
@@ -58,7 +60,7 @@ sudo /etc/init.d/memcached restart
 sudo "$(dirname "$0")/../../scripts/setup/terminate-psql-sessions" postgres zulip zulip_base
 
 # Remove and recreate the postgres database
-sudo pg_ctlcluster 9.3 main stop
+sudo pg_ctlcluster 9.5 main stop
-sudo pg_dropcluster 9.3 main
+sudo pg_dropcluster 9.5 main
-sudo rm -rf /etc/postgresql/9.3/main /var/lib/postgresql/9.3/main
+sudo rm -rf /etc/postgresql/9.5/main /var/lib/postgresql/9.5/main
-sudo pg_createcluster 9.3 main
+sudo pg_createcluster 9.5 main

tools/ci/success-http-headers.txt

@@ -2,7 +2,7 @@
 WARNING: no certificate subject alternative name matches
 	requested host name ‘localhost’.
   HTTP/1.1 302 Found
-  Server: nginx/1.4.6 (Ubuntu)
+  Server: nginx/1.10.3 (Ubuntu)
   Content-Type: text/html; charset=utf-8
   Content-Length: 0
   Connection: keep-alive
@@ -12,7 +12,7 @@ WARNING: no certificate subject alternative name matches
 Location: /login/ [following]
 Reusing existing connection to localhost:443.
   HTTP/1.1 200 OK
-  Server: nginx/1.4.6 (Ubuntu)
+  Server: nginx/1.10.3 (Ubuntu)
   Content-Type: text/html; charset=utf-8
   Content-Length: 6361
   Connection: keep-alive