parse_lsb_release: Use /etc/os-release instead of /etc/lsb-release.

scripts/lib/create-production-venv

@@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 if ZULIP_PATH not in sys.path:
     sys.path.append(ZULIP_PATH)
 
-from scripts.lib.zulip_tools import overwrite_symlink, run, parse_lsb_release
+from scripts.lib.zulip_tools import overwrite_symlink, run, parse_os_release
 from scripts.lib.setup_venv import (
     setup_virtualenv, VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
     FEDORA_VENV_DEPENDENCIES
@@ -19,8 +19,8 @@ parser.add_argument("deploy_path")
 args = parser.parse_args()
 
 # install dependencies for setting up the virtualenv
-distro_info = parse_lsb_release()
+distro_info = parse_os_release()
-vendor = distro_info['DISTRIB_ID']
+vendor = distro_info['ID']
 family = distro_info['DISTRIB_FAMILY']
 if family == 'debian':
     run(["apt-get", "-y", "install"] + VENV_DEPENDENCIES)
@@ -31,7 +31,7 @@ elif family == 'redhat':
         _VENV_DEPS = FEDORA_VENV_DEPENDENCIES
     run(["yum", "-y", "install"] + _VENV_DEPS)
 else:
-    print("Unsupported platform: {}".format(vendor))
+    print("Unsupported platform: {}".format(family))
     sys.exit(1)
 
 python_version = sys.version_info[0]

scripts/lib/create-thumbor-venv

@@ -8,7 +8,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 if ZULIP_PATH not in sys.path:
     sys.path.append(ZULIP_PATH)
 
-from scripts.lib.zulip_tools import run, parse_lsb_release
+from scripts.lib.zulip_tools import run, parse_os_release
 from scripts.lib.setup_venv import (
     setup_virtualenv, THUMBOR_VENV_DEPENDENCIES, YUM_THUMBOR_VENV_DEPENDENCIES
 )
@@ -18,15 +18,14 @@ parser.add_argument("deploy_path")
 args = parser.parse_args()
 
 # install dependencies for setting up the virtualenv
-distro_info = parse_lsb_release()
+distro_info = parse_os_release()
-vendor = distro_info['DISTRIB_ID']
 family = distro_info['DISTRIB_FAMILY']
 if family == 'debian':
     run(["apt-get", "-y", "install"] + THUMBOR_VENV_DEPENDENCIES)
 elif family == 'redhat':
     run(["yum", "-y", "install"] + YUM_THUMBOR_VENV_DEPENDENCIES)
 else:
-    print("Unsupported platform: {}".format(vendor))
+    print("Unsupported platform: {}".format(family))
     sys.exit(1)
 
 venv_name = "zulip-thumbor-venv"

scripts/lib/zulip_tools.py

@@ -337,8 +337,22 @@ def may_be_perform_purging(dirs_to_purge, dirs_to_keep, dir_type, dry_run, verbo
         if verbose:
             print("Keeping used %s: %s" % (dir_type, directory))
 
-def parse_lsb_release():
+def parse_os_release():
     # type: () -> Dict[str, str]
+    """
+    Example of the useful subset of the data:
+    {
+     'DISTRIB_FAMILY': 'debian'
+     'ID': 'ubuntu',
+     'VERSION_ID': '18.04',
+     'NAME': 'Ubuntu',
+     'VERSION': '18.04.3 LTS (Bionic Beaver)',
+     'PRETTY_NAME': 'Ubuntu 18.04.3 LTS',
+    }
+
+    VERSION_CODENAME (e.g. 'bionic') is nice and human-readable, but
+    we avoid using it, as it is not available on RHEL-based platforms.
+    """
     distro_info = {}  # type: Dict[str, str]
     if os.path.exists("/etc/redhat-release"):
         with open('/etc/redhat-release', 'r') as fp:
@@ -346,44 +360,30 @@ def parse_lsb_release():
         vendor = info[0]
         if vendor == 'CentOS':
             # E.g. "CentOS Linux release 7.5.1804 (Core)"
-            codename = vendor.lower() + info[3][0]
+            os_version = vendor.lower() + info[3][0]
         elif vendor == 'Fedora':
             # E.g. "Fedora release 29 (Twenty Nine)"
-            codename = vendor.lower() + info[2]
+            os_version = vendor.lower() + info[2]
         elif vendor == 'Red':
             # E.g. "Red Hat Enterprise Linux Server release 7.6 (Maipo)"
             vendor = 'RedHat'
-            codename = 'rhel' + info[6][0]  # 7
+            os_version = 'rhel' + info[6][0]  # 7
         distro_info = dict(
-            DISTRIB_CODENAME=codename,
+            VERSION_ID=os_version,
-            DISTRIB_ID=vendor,
+            ID=vendor,
             DISTRIB_FAMILY='redhat',
         )
         return distro_info
-    try:
+    with open('/etc/os-release', 'r') as fp:
-        # For performance reasons, we read /etc/lsb-release directly,
+        for line in fp:
-        # rather than using the lsb_release command; this saves ~50ms
+            line = line.strip()
-        # in several places in provisioning and the installer
+            if not line or line.startswith('#'):
-        with open('/etc/lsb-release', 'r') as fp:
+                # The line may be blank or a comment, see:
-            data = [line.strip().split('=') for line in fp]
+                # https://www.freedesktop.org/software/systemd/man/os-release.html
-        for k, v in data:
-            if k not in ['DISTRIB_CODENAME', 'DISTRIB_ID']:
-                # We only return to the caller the values that we get
-                # from lsb_release in the exception code path.
                 continue
-            distro_info[k] = v
+            k, v = line.split('=', 1)
+            [distro_info[k]] = shlex.split(v)
     distro_info['DISTRIB_FAMILY'] = 'debian'
-    except FileNotFoundError:
-        # Unfortunately, Debian stretch doesn't yet have an
-        # /etc/lsb-release, so we instead fetch the pieces of data
-        # that we use from the `lsb_release` command directly.
-        vendor = subprocess_text_output(["lsb_release", "-is"])
-        codename = subprocess_text_output(["lsb_release", "-cs"])
-        distro_info = dict(
-            DISTRIB_CODENAME=codename,
-            DISTRIB_ID=vendor,
-            DISTRIB_FAMILY='debian',
-        )
     return distro_info
 
 def file_or_package_hash_updated(paths, hash_name, is_force, package_versions=[]):

scripts/zulip-puppet-apply

@@ -5,7 +5,7 @@ import sys
 import subprocess
 import configparser
 import re
-from lib.zulip_tools import parse_lsb_release, assert_running_as_root
+from lib.zulip_tools import parse_os_release, assert_running_as_root
 
 assert_running_as_root()
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -21,8 +21,8 @@ config = configparser.RawConfigParser()
 config.read("/etc/zulip/zulip.conf")
 
 if not os.path.exists("/etc/puppet/hiera.yaml"):
-    codename = parse_lsb_release()['DISTRIB_CODENAME']
+    distro_info = parse_os_release()
-    if codename in ["stretch", "xenial"]:
+    if (distro_info['ID'], distro_info['VERSION_ID']) in [('debian', '9'), ('ubuntu', '16.04')]:
         # Suppress warnings in old puppet about hiera.yaml not existing.
         open("/etc/puppet/hiera.yaml", "a").close()
 

tools/lib/provision.py

@@ -14,7 +14,7 @@ ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__f
 
 sys.path.append(ZULIP_PATH)
 from scripts.lib.zulip_tools import run_as_root, ENDC, WARNING, \
-    get_dev_uuid_var_path, FAIL, parse_lsb_release, \
+    get_dev_uuid_var_path, FAIL, parse_os_release, \
     overwrite_symlink
 from scripts.lib.setup_venv import (
     VENV_DEPENDENCIES, REDHAT_VENV_DEPENDENCIES,
@@ -30,15 +30,15 @@ if TYPE_CHECKING:
     from typing_extensions import NoReturn
 
 SUPPORTED_PLATFORMS = {
-    "Ubuntu": [
+    "ubuntu": [
-        "xenial",
+        "16.04",  # xenial
-        "bionic",
+        "18.04",  # bionic
-        "cosmic",
+        "18.10",  # cosmic
-        "disco",
+        "19.04",  # disco
     ],
-    "Debian": [
+    "debian": [
-        "stretch",
+        "9",   # stretch
-        "buster",
+        "10",  # buster
     ],
     "CentOS": [
         "centos7",
@@ -104,18 +104,18 @@ else:
     sys.exit(1)
 
 # Ideally we wouldn't need to install a dependency here, before we
-# know the codename.
+# know the OS version.
 is_rhel_based = os.path.exists("/etc/redhat-release")
 if (not is_rhel_based) and (not os.path.exists("/usr/bin/lsb_release")):
     run_as_root(["apt-get", "install", "-y", "lsb-release"])
 
-distro_info = parse_lsb_release()
+distro_info = parse_os_release()
-vendor = distro_info['DISTRIB_ID']
+vendor = distro_info['ID']
-codename = distro_info['DISTRIB_CODENAME']
+os_version = distro_info['VERSION_ID']
 family = distro_info['DISTRIB_FAMILY']
-if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor]):
+if not (vendor in SUPPORTED_PLATFORMS and os_version in SUPPORTED_PLATFORMS[vendor]):
-    logging.critical("Unsupported platform: {} {}".format(vendor, codename))
+    logging.critical("Unsupported platform: {} {}".format(vendor, os_version))
-    if codename == 'trusty':
+    if vendor == 'ubuntu' and os_version == '14.04':
         print()
         print("Ubuntu Trusty reached end-of-life upstream and is no longer a supported platform for Zulip")
         if os.path.exists('/home/vagrant'):
@@ -124,17 +124,17 @@ if not (vendor in SUPPORTED_PLATFORMS and codename in SUPPORTED_PLATFORMS[vendor
     sys.exit(1)
 
 POSTGRES_VERSION_MAP = {
-    "stretch": "9.6",
+    ("debian", "9"): "9.6",
-    "buster": "11",
+    ("debian", "10"): "11",
-    "xenial": "9.5",
+    ("ubuntu", "16.04"): "9.5",
-    "bionic": "10",
+    ("ubuntu", "18.04"): "10",
-    "cosmic": "10",
+    ("ubuntu", "18.10"): "10",
-    "disco": "11",
+    ("ubuntu", "19.04"): "11",
-    "centos7": "10",
+    ("CentOS", "centos7"): "10",
-    "fedora29": "10",
+    ("Fedora", "fedora29"): "10",
-    "rhel7": "10",
+    ("RedHat", "rhel7"): "10",
 }
-POSTGRES_VERSION = POSTGRES_VERSION_MAP[codename]
+POSTGRES_VERSION = POSTGRES_VERSION_MAP[(vendor, os_version)]
 
 COMMON_DEPENDENCIES = [
     "memcached",
@@ -173,8 +173,7 @@ COMMON_YUM_DEPENDENCIES = COMMON_DEPENDENCIES + [
 
 BUILD_TSEARCH_FROM_SOURCE = False
 BUILD_PGROONGA_FROM_SOURCE = False
-if vendor in ["Ubuntu", "Debian"]:
+if vendor == "ubuntu" and os_version in ("18.10", "19.04"):
-    if codename in ("cosmic", "disco"):
     # For platforms without a tsearch-extras package distributed
     # from our PPA, we need to build from source.
     BUILD_TSEARCH_FROM_SOURCE = True
@@ -186,7 +185,7 @@ if vendor in ["Ubuntu", "Debian"]:
             "postgresql-server-dev-{0}",
         ]
     ]
-    elif codename == "buster":
+elif vendor == 'debian' and os_version == "10":
     # For platforms without a tsearch-extras package distributed
     # from our PPA or a pgroonga release, we need to build both
     # from source.
@@ -202,7 +201,7 @@ if vendor in ["Ubuntu", "Debian"]:
             "libmsgpack-dev",
         ]
     ]
-    else:
+elif vendor in ["ubuntu", "debian"]:
     SYSTEM_DEPENDENCIES = UBUNTU_COMMON_APT_DEPENDENCIES + [
         pkg.format(POSTGRES_VERSION) for pkg in [
             "postgresql-{0}",
@@ -255,7 +254,7 @@ def install_system_deps():
 
     if family == 'redhat':
         install_yum_deps(deps_to_install)
-    elif vendor in ["Debian", "Ubuntu"]:
+    elif vendor in ["debian", "ubuntu"]:
         install_apt_deps(deps_to_install)
     else:
         raise AssertionError("Invalid vendor")
@@ -352,7 +351,7 @@ def main(options):
 
     for apt_depedency in SYSTEM_DEPENDENCIES:
         sha_sum.update(apt_depedency.encode('utf8'))
-    if vendor in ["Ubuntu", "Debian"]:
+    if vendor in ["ubuntu", "debian"]:
         sha_sum.update(open('scripts/lib/setup-apt-repo', 'rb').read())
     else:
         # hash the content of setup-yum-repo and build-*

zerver/management/commands/backup.py

@@ -8,7 +8,7 @@ from django.conf import settings
 from django.db import connection
 from django.utils.timezone import now as timezone_now
 
-from scripts.lib.zulip_tools import parse_lsb_release, run, TIMESTAMP_FORMAT
+from scripts.lib.zulip_tools import parse_os_release, run, TIMESTAMP_FORMAT
 from version import ZULIP_VERSION
 from zerver.lib.management import ZulipBaseCommand
 from zerver.logging_handlers import try_git_describe
@@ -46,7 +46,7 @@ class Command(ZulipBaseCommand):
 
             with open(os.path.join(tmp, "zulip-backup", "os-version"), "w") as f:
                 print(
-                    "{DISTRIB_ID} {DISTRIB_CODENAME}".format(**parse_lsb_release()),
+                    "{ID} {VERSION_ID}".format(**parse_os_release()),
                     file=f,
                 )
             members.append("zulip-backup/os-version")