diff --git a/puppet/zulip_ops/manifests/base.pp b/puppet/zulip_ops/manifests/base.pp index 0a2d22f436..bfb92b9f54 100644 --- a/puppet/zulip_ops/manifests/base.pp +++ b/puppet/zulip_ops/manifests/base.pp @@ -17,12 +17,8 @@ class zulip_ops::base { # For managing our current Debian packages 'debian-goodies', # Needed for zulip-ec2-configure-network-interfaces - 'python3-six', - 'python-six', 'python3-boto3', - 'python-boto', # needed for postgres_common too 'python3-netifaces', - 'python-netifaces', # Popular editors 'vim', 'emacs-nox', diff --git a/requirements/common.in b/requirements/common.in index 20da2b2e46..234a0b376d 100644 --- a/requirements/common.in +++ b/requirements/common.in @@ -93,9 +93,6 @@ pytz # Needed for redis redis -# Needed for Python 2+3 compatibility -six - # Needed to parse source maps for error reporting sourcemap diff --git a/requirements/dev.txt b/requirements/dev.txt index 254334a0d1..7da03381a7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -911,7 +911,7 @@ sh==1.12.14 \ six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ - # via -r requirements/common.in, argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip + # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip snakeviz==2.1.0 \ --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \ --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \ diff --git a/requirements/prod.txt b/requirements/prod.txt index ace201a33c..60baf2b04f 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -661,7 +661,7 @@ s3transfer==0.3.3 \ six==1.15.0 \ --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \ - # via -r requirements/common.in, argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip + # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip social-auth-app-django==4.0.0 \ --hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \ --hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \ diff --git a/scripts/lib/install b/scripts/lib/install index 3e50f340bc..289b5a2fbe 100755 --- a/scripts/lib/install +++ b/scripts/lib/install @@ -275,7 +275,7 @@ fi if [ "$package_system" = apt ]; then if ! apt-get install -y \ puppet git curl wget jq \ - python3 python3-six crudini \ + python3 crudini \ "${ADDITIONAL_PACKAGES[@]}"; then set +x echo -e '\033[0;31m' >&2 @@ -287,7 +287,7 @@ if [ "$package_system" = apt ]; then elif [ "$package_system" = yum ]; then if ! yum install -y \ puppet git curl wget jq \ - python3 python3-six crudini \ + python3 crudini \ "${ADDITIONAL_PACKAGES[@]}"; then set +x echo -e '\033[0;31m' >&2 diff --git a/scripts/lib/setup_venv.py b/scripts/lib/setup_venv.py index c67640b4ce..3cba3b4c3f 100644 --- a/scripts/lib/setup_venv.py +++ b/scripts/lib/setup_venv.py @@ -21,7 +21,6 @@ VENV_DEPENDENCIES = [ "python3-dev", # Needed to install typed-ast dependency of mypy "python3-pip", "virtualenv", - "python3-six", "libxml2-dev", # Used for installing talon and python-xmlsec "libxslt1-dev", # Used for installing talon "libpq-dev", # Needed by psycopg2 @@ -60,13 +59,11 @@ COMMON_YUM_VENV_DEPENDENCIES = [ REDHAT_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [ "python36-devel", - "python36-six", "python-virtualenv", ] FEDORA_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [ "python3-pip", - "python3-six", "virtualenv", # see https://unix.stackexchange.com/questions/27877/install-virtualenv-on-fedora-16 ] diff --git a/tools/linter_lib/custom_check.py b/tools/linter_lib/custom_check.py index 4bb4dda8a7..e9d13af2f0 100644 --- a/tools/linter_lib/custom_check.py +++ b/tools/linter_lib/custom_check.py @@ -337,8 +337,7 @@ python_rules = RuleList( 'scripts/lib/zulip_tools.py', 'tools/lib/provision.py', 'zproject/dev_settings.py', - 'zproject/prod_settings_template.py', - 'zthumbor'}, + 'zproject/prod_settings_template.py'}, 'description': 'Comment-style variable type annotation. Use Python 3.6 style annotations instead.', 'good_lines': ['a: List[int] = []'], 'bad_lines': ['a = [] # type: List[int]']}, @@ -370,8 +369,6 @@ python_rules = RuleList( 'tools/lib', # TODO: Update our migrations from Text->str. 'zerver/migrations/', - # thumbor is (currently) python2 only - 'zthumbor/', }, 'description': "Now that we're a Python 3 only codebase, we don't need to use typing.Text. Please use str instead.", }, diff --git a/tools/test-install/prepare-base b/tools/test-install/prepare-base index 94ba65a93a..2c528b7247 100755 --- a/tools/test-install/prepare-base +++ b/tools/test-install/prepare-base @@ -53,7 +53,7 @@ run apt-get install -y --no-install-recommends \ hunspell-en-us supervisor libssl-dev puppet \ gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \ libldap2-dev libmemcached-dev \ - python-six libxml2-dev libxslt1-dev libpq-dev \ + libxml2-dev libxslt1-dev libpq-dev \ virtualenv \ "${extra_packages[@]}" diff --git a/zproject/dev_settings.py b/zproject/dev_settings.py index 4c903cb043..21848bb15a 100644 --- a/zproject/dev_settings.py +++ b/zproject/dev_settings.py @@ -2,7 +2,7 @@ import os import pwd from typing import Optional, Set, Tuple -from six.moves.urllib.parse import SplitResult +from scripts.lib.zulip_tools import deport ZULIP_ADMINISTRATOR = "desdemona+admin@zulip.com" @@ -39,11 +39,7 @@ else: 'zulip': EXTERNAL_HOST, } -# TODO: Replace with scripts.lib.zulip_tools.deport when this no longer needs to -# be Python 2 compatible for zthumbor. -r = SplitResult("", EXTERNAL_HOST, "", "", "") -assert r.hostname is not None -EXTERNAL_HOST_WITHOUT_PORT = "[" + r.hostname + "]" if ":" in r.hostname else r.hostname +EXTERNAL_HOST_WITHOUT_PORT = deport(EXTERNAL_HOST) ALLOWED_HOSTS = ['*'] diff --git a/zthumbor/loaders/helpers.py b/zthumbor/loaders/helpers.py index f3fa009066..cb19c0a9cd 100644 --- a/zthumbor/loaders/helpers.py +++ b/zthumbor/loaders/helpers.py @@ -1,47 +1,11 @@ -# This file is used by both Python 2.7 (thumbor) and 3 (zulip). -from __future__ import absolute_import - -import os import re -import sys -from typing import Any, Optional, Text, Tuple - -ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.append(ZULIP_PATH) - -# Piece of code below relating to secrets conf has been duplicated with that of -# django settings in zproject/settings.py -import six.moves.configparser - -DEPLOY_ROOT = os.path.join(os.path.realpath(os.path.dirname(__file__)), '..', '..') - -config_file = six.moves.configparser.RawConfigParser() -config_file.read("/etc/zulip/zulip.conf") - -# Whether this instance of Zulip is running in a production environment. -PRODUCTION = config_file.has_option('machine', 'deploy_type') -DEVELOPMENT = not PRODUCTION - -secrets_file = six.moves.configparser.RawConfigParser() -if PRODUCTION: - secrets_file.read("/etc/zulip/zulip-secrets.conf") -else: - secrets_file.read(os.path.join(DEPLOY_ROOT, "zproject/dev-secrets.conf")) - -def get_secret( - key: str, default_value: Optional[Any] = None, development_only: bool = False, -) -> Optional[Any]: - if development_only and PRODUCTION: - return default_value - if secrets_file.has_option('secrets', key): - return secrets_file.get('secrets', key) - return default_value +from typing import Tuple THUMBOR_EXTERNAL_TYPE = 'external' THUMBOR_S3_TYPE = 's3' THUMBOR_LOCAL_FILE_TYPE = 'local_file' -def separate_url_and_source_type(url: Text) -> Tuple[Text, Text]: +def separate_url_and_source_type(url: str) -> Tuple[str, str]: THUMBNAIL_URL_PATT = re.compile('^(?P.+)/source_type/(?P.+)') matches = THUMBNAIL_URL_PATT.match(url) assert matches is not None diff --git a/zthumbor/loaders/zloader.py b/zthumbor/loaders/zloader.py index 0bb255f048..3b713ff477 100644 --- a/zthumbor/loaders/zloader.py +++ b/zthumbor/loaders/zloader.py @@ -1,10 +1,9 @@ # See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html -from __future__ import absolute_import import base64 import logging +import urllib.parse -from six.moves import urllib from tc_aws.loaders import s3_loader from thumbor.context import Context from thumbor.loaders import LoaderResult, file_loader, https_loader diff --git a/zthumbor/thumbor.conf b/zthumbor/thumbor.conf index 319f817b6a..2cc14e00b5 100644 --- a/zthumbor/thumbor.conf +++ b/zthumbor/thumbor.conf @@ -1,19 +1,16 @@ +import configparser import os import sys -import subprocess - -import six.moves.configparser - -ZULIP_PATH = os.path.dirname(os.path.abspath('__file__')) +ZULIP_PATH = os.getcwd() # Thumbor doesn’t set __file__ when loading this sys.path.append(ZULIP_PATH) -from zthumbor.loaders.helpers import get_secret +from zproject.config import get_secret os.environ['AWS_ACCESS_KEY_ID'] = get_secret('s3_key', '') os.environ['AWS_SECRET_ACCESS_KEY'] = get_secret('s3_secret_key', '') -config_file = six.moves.configparser.RawConfigParser() +config_file = configparser.RawConfigParser() config_file.read("/etc/zulip/zulip.conf") # Whether this instance of Zulip is running in a production environment. @@ -656,17 +653,7 @@ RESULT_STORAGE_STORES_UNSAFE = True ################################### AWS S3 settings ############################ if not IS_LOCAL_STORAGE: - if PRODUCTION: - from zproject.prod_settings import S3_AUTH_UPLOADS_BUCKET - try: - from zproject.prod_settings import S3_REGION - except ImportError: - # If you upgraded from older Zulip, - # prod_settings_template.py didn't have S3_REGION before, - # so we should just assume the default value. - S3_REGION = '' - else: - from zproject.dev_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION + from zproject.configured_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION TC_AWS_REGION = S3_REGION # AWS Region