zthumbor: Remove Python 2 residue.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

puppet/zulip_ops/manifests/base.pp

@@ -17,12 +17,8 @@ class zulip_ops::base {
     # For managing our current Debian packages
     'debian-goodies',
     # Needed for zulip-ec2-configure-network-interfaces
-    'python3-six',
-    'python-six',
     'python3-boto3',
-    'python-boto', # needed for postgres_common too
     'python3-netifaces',
-    'python-netifaces',
     # Popular editors
     'vim',
     'emacs-nox',

requirements/common.in

@@ -93,9 +93,6 @@ pytz
 # Needed for redis
 redis
 
-# Needed for Python 2+3 compatibility
-six
-
 # Needed to parse source maps for error reporting
 sourcemap
 

requirements/dev.txt

@@ -911,7 +911,7 @@ sh==1.12.14 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via -r requirements/common.in, argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
+    # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
 snakeviz==2.1.0 \
     --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
     --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \

requirements/prod.txt

@@ -661,7 +661,7 @@ s3transfer==0.3.3 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via -r requirements/common.in, argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
+    # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
 social-auth-app-django==4.0.0 \
     --hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \
     --hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \

scripts/lib/install

@@ -275,7 +275,7 @@ fi
 if [ "$package_system" = apt ]; then
     if ! apt-get install -y \
          puppet git curl wget jq \
-         python3 python3-six crudini \
+         python3 crudini \
          "${ADDITIONAL_PACKAGES[@]}"; then
         set +x
         echo -e '\033[0;31m' >&2
@@ -287,7 +287,7 @@ if [ "$package_system" = apt ]; then
 elif [ "$package_system" = yum ]; then
     if ! yum install -y \
          puppet git curl wget jq \
-         python3 python3-six crudini \
+         python3 crudini \
          "${ADDITIONAL_PACKAGES[@]}"; then
         set +x
         echo -e '\033[0;31m' >&2

scripts/lib/setup_venv.py

@@ -21,7 +21,6 @@ VENV_DEPENDENCIES = [
     "python3-dev",          # Needed to install typed-ast dependency of mypy
     "python3-pip",
     "virtualenv",
-    "python3-six",
     "libxml2-dev",          # Used for installing talon and python-xmlsec
     "libxslt1-dev",         # Used for installing talon
     "libpq-dev",            # Needed by psycopg2
@@ -60,13 +59,11 @@ COMMON_YUM_VENV_DEPENDENCIES = [
 
 REDHAT_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [
     "python36-devel",
-    "python36-six",
     "python-virtualenv",
 ]
 
 FEDORA_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [
     "python3-pip",
-    "python3-six",
     "virtualenv",  # see https://unix.stackexchange.com/questions/27877/install-virtualenv-on-fedora-16
 ]
 

tools/linter_lib/custom_check.py

@@ -337,8 +337,7 @@ python_rules = RuleList(
                      'scripts/lib/zulip_tools.py',
                      'tools/lib/provision.py',
                      'zproject/dev_settings.py',
-                     'zproject/prod_settings_template.py',
+                     'zproject/prod_settings_template.py'},
-                     'zthumbor'},
          'description': 'Comment-style variable type annotation. Use Python 3.6 style annotations instead.',
          'good_lines': ['a: List[int] = []'],
          'bad_lines': ['a = []  # type: List[int]']},
@@ -370,8 +369,6 @@ python_rules = RuleList(
              'tools/lib',
              # TODO: Update our migrations from Text->str.
              'zerver/migrations/',
-             # thumbor is (currently) python2 only
-             'zthumbor/',
          },
          'description': "Now that we're a Python 3 only codebase, we don't need to use typing.Text. Please use str instead.",
          },

tools/test-install/prepare-base

@@ -53,7 +53,7 @@ run apt-get install -y --no-install-recommends \
   hunspell-en-us supervisor libssl-dev puppet \
   gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
   libldap2-dev libmemcached-dev \
-  python-six libxml2-dev libxslt1-dev libpq-dev \
+  libxml2-dev libxslt1-dev libpq-dev \
   virtualenv \
   "${extra_packages[@]}"
 

zproject/dev_settings.py

@@ -2,7 +2,7 @@ import os
 import pwd
 from typing import Optional, Set, Tuple
 
-from six.moves.urllib.parse import SplitResult
+from scripts.lib.zulip_tools import deport
 
 ZULIP_ADMINISTRATOR = "desdemona+admin@zulip.com"
 
@@ -39,11 +39,7 @@ else:
         'zulip': EXTERNAL_HOST,
     }
 
-# TODO: Replace with scripts.lib.zulip_tools.deport when this no longer needs to
+EXTERNAL_HOST_WITHOUT_PORT = deport(EXTERNAL_HOST)
-# be Python 2 compatible for zthumbor.
-r = SplitResult("", EXTERNAL_HOST, "", "", "")
-assert r.hostname is not None
-EXTERNAL_HOST_WITHOUT_PORT = "[" + r.hostname + "]" if ":" in r.hostname else r.hostname
 
 ALLOWED_HOSTS = ['*']
 

zthumbor/loaders/helpers.py

@@ -1,47 +1,11 @@
-# This file is used by both Python 2.7 (thumbor) and 3 (zulip).
-from __future__ import absolute_import
-
-import os
 import re
-import sys
+from typing import Tuple
-from typing import Any, Optional, Text, Tuple
-
-ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(ZULIP_PATH)
-
-# Piece of code below relating to secrets conf has been duplicated with that of
-# django settings in zproject/settings.py
-import six.moves.configparser
-
-DEPLOY_ROOT = os.path.join(os.path.realpath(os.path.dirname(__file__)), '..', '..')
-
-config_file = six.moves.configparser.RawConfigParser()
-config_file.read("/etc/zulip/zulip.conf")
-
-# Whether this instance of Zulip is running in a production environment.
-PRODUCTION = config_file.has_option('machine', 'deploy_type')
-DEVELOPMENT = not PRODUCTION
-
-secrets_file = six.moves.configparser.RawConfigParser()
-if PRODUCTION:
-    secrets_file.read("/etc/zulip/zulip-secrets.conf")
-else:
-    secrets_file.read(os.path.join(DEPLOY_ROOT, "zproject/dev-secrets.conf"))
-
-def get_secret(
-    key: str, default_value: Optional[Any] = None, development_only: bool = False,
-) -> Optional[Any]:
-    if development_only and PRODUCTION:
-        return default_value
-    if secrets_file.has_option('secrets', key):
-        return secrets_file.get('secrets', key)
-    return default_value
 
 THUMBOR_EXTERNAL_TYPE = 'external'
 THUMBOR_S3_TYPE = 's3'
 THUMBOR_LOCAL_FILE_TYPE = 'local_file'
 
-def separate_url_and_source_type(url: Text) -> Tuple[Text, Text]:
+def separate_url_and_source_type(url: str) -> Tuple[str, str]:
     THUMBNAIL_URL_PATT = re.compile('^(?P<actual_url>.+)/source_type/(?P<source_type>.+)')
     matches = THUMBNAIL_URL_PATT.match(url)
     assert matches is not None

zthumbor/loaders/zloader.py

@@ -1,10 +1,9 @@
 # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html
-from __future__ import absolute_import
 
 import base64
 import logging
+import urllib.parse
 
-from six.moves import urllib
 from tc_aws.loaders import s3_loader
 from thumbor.context import Context
 from thumbor.loaders import LoaderResult, file_loader, https_loader

zthumbor/thumbor.conf

@@ -1,19 +1,16 @@
+import configparser
 import os
 import sys
-import subprocess
-
-import six.moves.configparser
-
-ZULIP_PATH = os.path.dirname(os.path.abspath('__file__'))
 
+ZULIP_PATH = os.getcwd()  # Thumbor doesn’t set __file__ when loading this
 sys.path.append(ZULIP_PATH)
 
-from zthumbor.loaders.helpers import get_secret
+from zproject.config import get_secret
 
 os.environ['AWS_ACCESS_KEY_ID'] = get_secret('s3_key', '')
 os.environ['AWS_SECRET_ACCESS_KEY'] = get_secret('s3_secret_key', '')
 
-config_file = six.moves.configparser.RawConfigParser()
+config_file = configparser.RawConfigParser()
 config_file.read("/etc/zulip/zulip.conf")
 
 # Whether this instance of Zulip is running in a production environment.
@@ -656,17 +653,7 @@ RESULT_STORAGE_STORES_UNSAFE = True
 ################################### AWS S3 settings ############################
 
 if not IS_LOCAL_STORAGE:
-    if PRODUCTION:
+    from zproject.configured_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION
-        from zproject.prod_settings import S3_AUTH_UPLOADS_BUCKET
-        try:
-            from zproject.prod_settings import S3_REGION
-        except ImportError:
-            # If you upgraded from older Zulip,
-            # prod_settings_template.py didn't have S3_REGION before,
-            # so we should just assume the default value.
-            S3_REGION = ''
-    else:
-        from zproject.dev_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION
 
     TC_AWS_REGION = S3_REGION  # AWS Region