paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-10-23 04:52:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

zthumbor: Remove Python 2 residue.

Browse Source 
Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg
2020-07-06 16:26:50 -07:00
committed by Anders Kaseorg
parent bd6fe786cf
commit 9900298315
12 changed files with 16 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
puppet/zulip_ops/manifests/base.pp
View File

@@ -17,12 +17,8 @@ class zulip_ops::base {
# For managing our current Debian packages
'debian-goodies',
# Needed for zulip-ec2-configure-network-interfaces
'python3-six',
'python-six',
'python3-boto3',
'python-boto', # needed for postgres_common too
'python3-netifaces',
'python-netifaces',
# Popular editors
'vim',
'emacs-nox',

3
requirements/common.in
View File

@@ -93,9 +93,6 @@ pytz
# Needed for redis
redis
# Needed for Python 2+3 compatibility
six
# Needed to parse source maps for error reporting
sourcemap

2
requirements/dev.txt
View File

@@ -911,7 +911,7 @@ sh==1.12.14 \
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via -r requirements/common.in, argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
# via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
snakeviz==2.1.0 \
--hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
--hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \

2
requirements/prod.txt
View File

@@ -661,7 +661,7 @@ s3transfer==0.3.3 \
six==1.15.0 \
--hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
--hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
# via -r requirements/common.in, argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
# via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
social-auth-app-django==4.0.0 \
--hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \
--hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \

4
scripts/lib/install
View File

@@ -275,7 +275,7 @@ fi
if [ "$package_system" = apt ]; then
if ! apt-get install -y \
puppet git curl wget jq \
python3 python3-six crudini \
python3 crudini \
"${ADDITIONAL_PACKAGES[@]}"; then
set +x
echo -e '\033[0;31m' >&2
@@ -287,7 +287,7 @@ if [ "$package_system" = apt ]; then
elif [ "$package_system" = yum ]; then
if ! yum install -y \
puppet git curl wget jq \
python3 python3-six crudini \
python3 crudini \
"${ADDITIONAL_PACKAGES[@]}"; then
set +x
echo -e '\033[0;31m' >&2

3
scripts/lib/setup_venv.py
View File

@@ -21,7 +21,6 @@ VENV_DEPENDENCIES = [
"python3-dev", # Needed to install typed-ast dependency of mypy
"python3-pip",
"virtualenv",
"python3-six",
"libxml2-dev", # Used for installing talon and python-xmlsec
"libxslt1-dev", # Used for installing talon
"libpq-dev", # Needed by psycopg2
@@ -60,13 +59,11 @@ COMMON_YUM_VENV_DEPENDENCIES = [
REDHAT_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [
"python36-devel",
"python36-six",
"python-virtualenv",
]
FEDORA_VENV_DEPENDENCIES = COMMON_YUM_VENV_DEPENDENCIES + [
"python3-pip",
"python3-six",
"virtualenv", # see https://unix.stackexchange.com/questions/27877/install-virtualenv-on-fedora-16
]

5
tools/linter_lib/custom_check.py
View File

@@ -337,8 +337,7 @@ python_rules = RuleList(
'scripts/lib/zulip_tools.py',
'tools/lib/provision.py',
'zproject/dev_settings.py',
'zproject/prod_settings_template.py',
'zthumbor'},
'zproject/prod_settings_template.py'},
'description': 'Comment-style variable type annotation. Use Python 3.6 style annotations instead.',
'good_lines': ['a: List[int] = []'],
'bad_lines': ['a = [] # type: List[int]']},
@@ -370,8 +369,6 @@ python_rules = RuleList(
'tools/lib',
# TODO: Update our migrations from Text->str.
'zerver/migrations/',
# thumbor is (currently) python2 only
'zthumbor/',
},
'description': "Now that we're a Python 3 only codebase, we don't need to use typing.Text. Please use str instead.",
},

2
tools/test-install/prepare-base
View File

@@ -53,7 +53,7 @@ run apt-get install -y --no-install-recommends \
hunspell-en-us supervisor libssl-dev puppet \
gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
libldap2-dev libmemcached-dev \
python-six libxml2-dev libxslt1-dev libpq-dev \
libxml2-dev libxslt1-dev libpq-dev \
virtualenv \
"${extra_packages[@]}"

8
zproject/dev_settings.py
View File

@@ -2,7 +2,7 @@ import os
import pwd
from typing import Optional, Set, Tuple
from six.moves.urllib.parse import SplitResult
from scripts.lib.zulip_tools import deport
ZULIP_ADMINISTRATOR = "desdemona+admin@zulip.com"
@@ -39,11 +39,7 @@ else:
'zulip': EXTERNAL_HOST,
}
# TODO: Replace with scripts.lib.zulip_tools.deport when this no longer needs to
# be Python 2 compatible for zthumbor.
r = SplitResult("", EXTERNAL_HOST, "", "", "")
assert r.hostname is not None
EXTERNAL_HOST_WITHOUT_PORT = "[" + r.hostname + "]" if ":" in r.hostname else r.hostname
EXTERNAL_HOST_WITHOUT_PORT = deport(EXTERNAL_HOST)
ALLOWED_HOSTS = ['*']

40
zthumbor/loaders/helpers.py
View File

@@ -1,47 +1,11 @@
# This file is used by both Python 2.7 (thumbor) and 3 (zulip).
from __future__ import absolute_import
import os
import re
import sys
from typing import Any, Optional, Text, Tuple
ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.append(ZULIP_PATH)
# Piece of code below relating to secrets conf has been duplicated with that of
# django settings in zproject/settings.py
import six.moves.configparser
DEPLOY_ROOT = os.path.join(os.path.realpath(os.path.dirname(__file__)), '..', '..')
config_file = six.moves.configparser.RawConfigParser()
config_file.read("/etc/zulip/zulip.conf")
# Whether this instance of Zulip is running in a production environment.
PRODUCTION = config_file.has_option('machine', 'deploy_type')
DEVELOPMENT = not PRODUCTION
secrets_file = six.moves.configparser.RawConfigParser()
if PRODUCTION:
secrets_file.read("/etc/zulip/zulip-secrets.conf")
else:
secrets_file.read(os.path.join(DEPLOY_ROOT, "zproject/dev-secrets.conf"))
def get_secret(
key: str, default_value: Optional[Any] = None, development_only: bool = False,
) -> Optional[Any]:
if development_only and PRODUCTION:
return default_value
if secrets_file.has_option('secrets', key):
return secrets_file.get('secrets', key)
return default_value
from typing import Tuple
THUMBOR_EXTERNAL_TYPE = 'external'
THUMBOR_S3_TYPE = 's3'
THUMBOR_LOCAL_FILE_TYPE = 'local_file'
def separate_url_and_source_type(url: Text) -> Tuple[Text, Text]:
def separate_url_and_source_type(url: str) -> Tuple[str, str]:
THUMBNAIL_URL_PATT = re.compile('^(?P<actual_url>.+)/source_type/(?P<source_type>.+)')
matches = THUMBNAIL_URL_PATT.match(url)
assert matches is not None

3
zthumbor/loaders/zloader.py
View File

@@ -1,10 +1,9 @@
# See https://zulip.readthedocs.io/en/latest/subsystems/thumbnailing.html
from __future__ import absolute_import
import base64
import logging
import urllib.parse
from six.moves import urllib
from tc_aws.loaders import s3_loader
from thumbor.context import Context
from thumbor.loaders import LoaderResult, file_loader, https_loader

23
zthumbor/thumbor.conf
View File

@@ -1,19 +1,16 @@
import configparser
import os
import sys
import subprocess
import six.moves.configparser
ZULIP_PATH = os.path.dirname(os.path.abspath('__file__'))
ZULIP_PATH = os.getcwd() # Thumbor doesn’t set __file__ when loading this
sys.path.append(ZULIP_PATH)
from zthumbor.loaders.helpers import get_secret
from zproject.config import get_secret
os.environ['AWS_ACCESS_KEY_ID'] = get_secret('s3_key', '')
os.environ['AWS_SECRET_ACCESS_KEY'] = get_secret('s3_secret_key', '')
config_file = six.moves.configparser.RawConfigParser()
config_file = configparser.RawConfigParser()
config_file.read("/etc/zulip/zulip.conf")
# Whether this instance of Zulip is running in a production environment.
@@ -656,17 +653,7 @@ RESULT_STORAGE_STORES_UNSAFE = True
################################### AWS S3 settings ############################
if not IS_LOCAL_STORAGE:
if PRODUCTION:
from zproject.prod_settings import S3_AUTH_UPLOADS_BUCKET
try:
from zproject.prod_settings import S3_REGION
except ImportError:
# If you upgraded from older Zulip,
# prod_settings_template.py didn't have S3_REGION before,
# so we should just assume the default value.
S3_REGION = ''
else:
from zproject.dev_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION
from zproject.configured_settings import S3_AUTH_UPLOADS_BUCKET, S3_REGION
TC_AWS_REGION = S3_REGION # AWS Region