From 8aaf34d06d7c73123ad88755dc59edb76f409a94 Mon Sep 17 00:00:00 2001 From: rht Date: Fri, 12 May 2017 07:30:38 +0700 Subject: [PATCH] Lock the recursive requirements in common.txt via pip-compile. The commit is composed of: (1) distill out top-level dependencies in common.txt, (2) add -e flag to the vcs-based packages because pip-compile can't do without, (3) pip-compile/generate the locked files then remove the -e flags from the lockfile, (4) pin pathlib2 to dev.txt because it turns out it is a direct requirement of documentation_crawler, (5) document the structure and add an automation script (6) remove cryptography==1.9 from requirements/scrapy.txt since cryptography is automatically added from pyopenssl (7) add sed command to remove future/futures from the generated lock file in python3 (this should have been automatically handled by pip-compile, so pending for the feature from pip-compile) Tweaked by tabbott to update PROVISION_VERSION and add a missing `first` dependency. --- requirements/README.md | 32 +++++++++ requirements/common.txt | 29 ++------- requirements/dev.txt | 9 +++ requirements/emailmirror.txt | 2 +- requirements/py2_common_lock.txt | 107 +++++++++++++++++++++++++++++++ requirements/py2_dev.txt | 2 +- requirements/py2_prod.txt | 2 +- requirements/py3_common_lock.txt | 98 ++++++++++++++++++++++++++++ requirements/py3_dev.txt | 2 +- requirements/py3_prod.txt | 2 +- requirements/scrapy.txt | 5 -- tools/update-locked-requirements | 25 ++++++++ version.py | 2 +- 13 files changed, 284 insertions(+), 33 deletions(-) create mode 100644 requirements/README.md create mode 100644 requirements/py2_common_lock.txt create mode 100644 requirements/py3_common_lock.txt create mode 100755 tools/update-locked-requirements diff --git a/requirements/README.md b/requirements/README.md new file mode 100644 index 0000000000..9115f7fe4e --- /dev/null +++ b/requirements/README.md @@ -0,0 +1,32 @@ +The dependency graph of the requirements is as follows: + +``` +py3_dev py3_prod py2_prod ++ + + + + + + +| | +->py3_common<-+ v v +>py2_common+----+ +| | | + prod | + | +| | | v | v | +| | |py3_socialauth |py2_socialauth| +| v | | | +| >dev +--->common<------------+ | +| | + + | +| | v +-->emailmirror | +| |docs,moto,py3k,twisted | +| +-----------------------------------+py2_dev<+ +v +mypy +``` + +Of the files, only py2_dev, py2_prod, py3_dev, py3_prod, and mypy have been +used in the install scripts directly. The rest are implicit dependencies. + +py2_common and py3_common are locked. + +Steps to update a lock file, e.g. to update ipython from 5.3.0 to 6.0.0 in +common.txt and propagate it py2_common_lock.txt and py3_common_lock: +0. Replace `ipython==5.4.1` with `ipython==6.0.0` in common.txt +1. Run './tools/update-locked-requirements' + +The reason the steps to remove the `-e` is necessary is because `pip-compile` +doesn't support installing from vcs without `-e` yet. +You may track the ongoing progress here https://github.com/jazzband/pip-tools/issues/355. diff --git a/requirements/common.txt b/requirements/common.txt index b18ac8da4d..c49ca27f06 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,4 +1,3 @@ --r ipython.txt # Django itself; we use a slightly patched version Django==1.11.2 @@ -8,13 +7,14 @@ Jinja2==2.9.6 # Needed for markdown processing Markdown==2.6.8 MarkupSafe==1.0 +Pygments==2.2.0 + +# Needed for manage.py +ipython==5.4.1 # Needed for Image Processing Pillow==4.1.1 -# Needed for markdown processing -Pygments==2.2.0 - # Needed for building complex DB queries SQLAlchemy==1.1.10 @@ -55,9 +55,6 @@ django-statsd-mozilla==0.3.16 docopt==0.6.2 fonttools==3.13.1 -# Needed for mock 2.0.0 -funcsigs==1.0.2 - # Needed for Android push notifications python-gcm==0.4 @@ -86,9 +83,6 @@ oauthlib==2.0.2 # Needed by requests to send https request to some sites. ndg-httpsclient==0.4.2 -# Needed for mock>1.1.2 -pbr==3.0.1 - # Needed to access rabbitmq pika==0.10.0 @@ -99,9 +93,6 @@ pyasn1==0.2.3 pyasn1-modules==0.0.9 pycrypto==2.6.1 -# Dependency of argon2-cffi -pycparser==2.17 - # Needed for memcached usage pylibmc==1.5.2 @@ -122,9 +113,6 @@ rsa==3.4.2 # Dependencies of our API simplejson==3.10.0 -# Needed for Tornado>3.2 -singledispatch==3.4.0.3 - # Needed for Python 2+3 compatibility six==1.10.0 smmap==0.9.0 @@ -135,9 +123,6 @@ sockjs-tornado==1.0.3 # Needed to parse source maps for error reporting sourcemap==0.2.0 -# Needed for django-statsd-mozilla -statsd==3.2.1 - # Tornado used for server->client push system tornado==4.5.1 @@ -156,7 +141,7 @@ django-pipeline==1.6.13 django-webpack-loader==0.5.0 # Needed for iOS push notifications -git+https://github.com/sharmaeklavya2/PyAPNs.git@remove-simplejson#egg=apns==2.0.1 +-e git+https://github.com/sharmaeklavya2/PyAPNs.git@remove-simplejson#egg=apns==2.0.1 python-twitter==3.3 @@ -167,11 +152,11 @@ python-twitter==3.3 polib==1.0.8 # Needed for cloning virtual environments -git+https://github.com/umairwaheed/virtualenv-clone.git@short-version#egg=virtualenv-clone==0.2.6 +-e git+https://github.com/umairwaheed/virtualenv-clone.git@short-version#egg=virtualenv-clone==0.2.6 # Needed for reading json as stream ijson==2.3 # Needed for link preview beautifulsoup4==4.6.0 -git+https://github.com/rafaelmartins/pyoembed.git@eb9901917c2a44b49e2887c077ead84a722c50dc#egg=pyoembed +-e git+https://github.com/rafaelmartins/pyoembed.git@eb9901917c2a44b49e2887c077ead84a722c50dc#egg=pyoembed diff --git a/requirements/dev.txt b/requirements/dev.txt index a45d774208..506be52687 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -33,6 +33,15 @@ transifex-client==0.12.4 # Dependency of transifex-client urllib3==1.21.1 +# Needed for documentation_crawler +pathlib2==2.2.1 +#Needed for pathlib2==2.2.1 +scandir==1.5 + +# Needed for updating the locked pip dependencies +first==2.0.1 +pip-tools==1.9.0 + # Install the Zulip API bindings into the virtualenv; note that this # won't be automatically updated for new versions of api/. This is # used in development both for tests and as a convenient for bot diff --git a/requirements/emailmirror.txt b/requirements/emailmirror.txt index 0e3d160784..8e542ab0d2 100644 --- a/requirements/emailmirror.txt +++ b/requirements/emailmirror.txt @@ -1,4 +1,4 @@ -git+https://github.com/umairwaheed/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.11 +-e git+https://github.com/umairwaheed/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.11 cchardet==2.1.0 cssselect==1.0.1 lxml==3.8.0 diff --git a/requirements/py2_common_lock.txt b/requirements/py2_common_lock.txt new file mode 100644 index 0000000000..0f9ff7ae26 --- /dev/null +++ b/requirements/py2_common_lock.txt @@ -0,0 +1,107 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/py2_common_lock.txt requirements/py2_common.txt +# +git+https://github.com/sharmaeklavya2/PyAPNs.git@remove-simplejson#egg=apns==2.0.1 +git+https://github.com/rafaelmartins/pyoembed.git@eb9901917c2a44b49e2887c077ead84a722c50dc#egg=pyoembed +git+https://github.com/umairwaheed/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.11 +git+https://github.com/umairwaheed/virtualenv-clone.git@short-version#egg=virtualenv-clone==0.2.6 +argon2-cffi==16.3.0 +asn1crypto==0.22.0 # via cryptography +backports-abc==0.5 +backports.shutil_get_terminal_size==1.0.0 +backports.ssl-match-hostname==3.5.0.1 +beautifulsoup4==4.6.0 +boto==2.47.0 +cchardet==2.1.0 +certifi==2017.4.17 +cffi==1.10.0 +chardet==3.0.4 +cryptography==1.8.1 # via pyopenssl +cssselect==1.0.1 +decorator==4.0.11 # via ipython, traitlets +defusedxml==0.5.0 +diff-match-patch==20121119 +django-auth-ldap==1.2.12 +django-bitfield==1.9.3 +django-pipeline==1.6.13 +django-statsd-mozilla==0.3.16 +django-webpack-loader==0.5.0 +django==1.11.2 +docopt==0.6.2 +enum34==1.1.6 +fonttools==3.13.1 +funcsigs==1.0.2 # via mock +future==0.16.0 # via python-twitter +futures==3.1.1 # via django-pipeline +gitdb==0.6.4 +google-api-python-client==1.6.2 +html2text==2016.9.19 +httplib2==0.10.3 +idna==2.5 +ijson==2.3 +ipaddress==1.0.18 # via cryptography +ipython-genutils==0.2.0 # via traitlets +ipython==5.4.1 +jinja2==2.9.6 +lxml==3.8.0 +markdown-include==0.5.1 +markdown==2.6.8 # via markdown-include +markupsafe==1.0 +mock==2.0.0 +ndg-httpsclient==0.4.2 +oauth2client==4.1.1 +oauthlib==2.0.2 +olefile==0.44 # via pillow +packaging==16.8 # via cryptography +pathlib2==2.2.1 # via ipython, pickleshare +pbr==3.0.0 # via mock +pexpect==4.2.1 # via ipython +pickleshare==0.7.4 # via ipython +pika==0.10.0 +pillow==4.1.1 +polib==1.0.8 +prompt-toolkit==1.0.14 # via ipython +psycopg2==2.7.1 +ptyprocess==0.5.1 # via pexpect +pyasn1-modules==0.0.9 +pyasn1==0.2.3 +pycparser==2.17 # via cffi +pycrypto==2.6.1 +pydns==2.3.6 +pygments==2.2.0 # via ipython +pyjwt==1.5.0 +pylibmc==1.5.2 +pyopenssl==17.0.0 # via ndg-httpsclient +pyparsing==2.2.0 # via packaging +python-gcm==0.4 +python-ldap==2.4.39 +python-openid==2.2.5 +python-twitter==3.3 +pytz==2017.2 +redis==2.10.5 +regex==2017.6.7 +requests-oauthlib==0.8.0 +requests==2.18.1 +rsa==3.4.2 +scandir==1.5 # via pathlib2 +simplegeneric==0.8.1 # via ipython +simplejson==3.10.0 +singledispatch==3.4.0.3 # via tornado +six==1.10.0 +smmap==0.9.0 +social-auth-app-django==1.2.0 +social-auth-core==1.4.0 +sockjs-tornado==1.0.3 +sourcemap==0.2.0 +sqlalchemy==1.1.10 +statsd==2.1.2 # via django-statsd-mozilla +tornado==4.5.1 +traitlets==4.3.2 # via ipython +typing==3.6.1 +ujson==1.35 +uritemplate==3.0.0 +urllib3==1.21.1 +wcwidth==0.1.7 # via prompt-toolkit diff --git a/requirements/py2_dev.txt b/requirements/py2_dev.txt index a4199d84a6..4ec260c7e4 100644 --- a/requirements/py2_dev.txt +++ b/requirements/py2_dev.txt @@ -1,2 +1,2 @@ --r py2_common.txt +-r py2_common_lock.txt -r dev.txt diff --git a/requirements/py2_prod.txt b/requirements/py2_prod.txt index e38fefb7ce..1ff2ed3875 100644 --- a/requirements/py2_prod.txt +++ b/requirements/py2_prod.txt @@ -1,3 +1,3 @@ --r py2_common.txt +-r py2_common_lock.txt -r prod.txt flup==1.0.2 diff --git a/requirements/py3_common_lock.txt b/requirements/py3_common_lock.txt new file mode 100644 index 0000000000..27d534c8ac --- /dev/null +++ b/requirements/py3_common_lock.txt @@ -0,0 +1,98 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/py3_common_lock.txt requirements/py3_common.txt +# +git+https://github.com/sharmaeklavya2/PyAPNs.git@remove-simplejson#egg=apns==2.0.1 +git+https://github.com/rafaelmartins/pyoembed.git@eb9901917c2a44b49e2887c077ead84a722c50dc#egg=pyoembed +git+https://github.com/umairwaheed/talon.git@7d8bdc4dbcfcc5a73298747293b99fe53da55315#egg=talon==1.2.11 +git+https://github.com/umairwaheed/virtualenv-clone.git@short-version#egg=virtualenv-clone==0.2.6 +argon2-cffi==16.3.0 +asn1crypto==0.22.0 # via cryptography +backports-abc==0.5 +backports.ssl-match-hostname==3.5.0.1 +beautifulsoup4==4.6.0 +boto==2.47.0 +cchardet==2.1.0 +certifi==2017.4.17 +cffi==1.10.0 +chardet==3.0.4 +cryptography==1.8.1 # via pyopenssl +cssselect==1.0.1 +decorator==4.0.11 # via ipython, traitlets +defusedxml==0.5.0 +diff-match-patch==20121119 +django-auth-ldap==1.2.12 +django-bitfield==1.9.3 +django-pipeline==1.6.13 +django-statsd-mozilla==0.3.16 +django-webpack-loader==0.5.0 +django==1.11.2 +docopt==0.6.2 +fonttools==3.13.1 +gitdb==0.6.4 +google-api-python-client==1.6.2 +html2text==2016.9.19 +httplib2==0.10.3 +idna==2.5 +ijson==2.3 +ipython-genutils==0.2.0 # via traitlets +ipython==5.4.1 +jinja2==2.9.6 +lxml==3.8.0 +markdown-include==0.5.1 +markdown==2.6.8 # via markdown-include +markupsafe==1.0 +mock==2.0.0 +ndg-httpsclient==0.4.2 +oauth2client==4.1.1 +oauthlib==2.0.2 +olefile==0.44 # via pillow +packaging==16.8 # via cryptography +pbr==3.0.0 # via mock +pexpect==4.2.1 # via ipython +pickleshare==0.7.4 # via ipython +pika==0.10.0 +pillow==4.1.1 +polib==1.0.8 +prompt-toolkit==1.0.14 # via ipython +psycopg2==2.7.1 +ptyprocess==0.5.1 # via pexpect +py3dns==3.1.0 +pyasn1-modules==0.0.9 +pyasn1==0.2.3 +pycparser==2.17 # via cffi +pycrypto==2.6.1 +pygments==2.2.0 # via ipython +pyjwt==1.5.0 +pyldap==2.4.25.1 +pylibmc==1.5.2 +pyopenssl==17.0.0 # via ndg-httpsclient +pyparsing==2.2.0 # via packaging +python-gcm==0.4 +python-twitter==3.3 +python3-openid==3.1.0 +pytz==2017.2 +redis==2.10.5 +regex==2017.6.7 +requests-oauthlib==0.8.0 +requests==2.18.1 +rsa==3.4.2 +simplegeneric==0.8.1 # via ipython +simplejson==3.10.0 +six==1.10.0 +smmap==0.9.0 +social-auth-app-django==1.2.0 +social-auth-core==1.4.0 +sockjs-tornado==1.0.3 +sourcemap==0.2.0 +sqlalchemy==1.1.10 +statsd==2.1.2 # via django-statsd-mozilla +tornado==4.5.1 +traitlets==4.3.2 # via ipython +typing==3.6.1 +ujson==1.35 +uritemplate==3.0.0 +urllib3==1.21.1 +wcwidth==0.1.7 # via prompt-toolkit diff --git a/requirements/py3_dev.txt b/requirements/py3_dev.txt index 1495c6067b..087511c8d8 100644 --- a/requirements/py3_dev.txt +++ b/requirements/py3_dev.txt @@ -1,3 +1,3 @@ --r py3_common.txt +-r py3_common_lock.txt -r dev.txt -r mypy.txt diff --git a/requirements/py3_prod.txt b/requirements/py3_prod.txt index 538b48f421..3f5053cc9c 100644 --- a/requirements/py3_prod.txt +++ b/requirements/py3_prod.txt @@ -1,3 +1,3 @@ --r py3_common.txt +-r py3_common_lock.txt -r prod.txt flup==1.0.3.dev20161029 diff --git a/requirements/scrapy.txt b/requirements/scrapy.txt index a8f7c89cc4..d6d76e5572 100644 --- a/requirements/scrapy.txt +++ b/requirements/scrapy.txt @@ -1,11 +1,6 @@ Scrapy==1.4.0 -cryptography==1.9 asn1crypto==0.22.0 -# Required by cryptography, only for Python 2 (should be removed once we -# migrate to Python 3): -ipaddress==1.0.18 - parsel==1.2.0 pyOpenSSL==17.0.0 w3lib==1.17.0 diff --git a/tools/update-locked-requirements b/tools/update-locked-requirements new file mode 100755 index 0000000000..8b28d46939 --- /dev/null +++ b/tools/update-locked-requirements @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -e + +# Check if zulip-.*venv in particular has been installed +# It is sufficient to check just for /srv/zulip-py3-venv because it must be +# present in both cases of py2 and py3 +if [ ! -d /srv/zulip-py3-venv ]; then + ./tools/setup/setup_venvs.py +fi + +source /srv/zulip-venv/bin/activate # Activate python2 virtualenv +pip-compile --output-file requirements/py2_common_lock.txt requirements/py2_common.txt +# Remove the editable flag in the lock file for safer build +sed -i 's/-e //' requirements/py2_common_lock.txt +deactivate + +source /srv/zulip-py3-venv/bin/activate # Activate python3 virtualenv +pip-compile --output-file requirements/py3_common_lock.txt requirements/py3_common.txt +# Remove the editable flag in the lock file for safer build +sed -i 's/-e //' requirements/py3_common_lock.txt +# pip-tools bug; future, futures are obsolete in python3 +sed -i '/futures==/d' requirements/py3_common_lock.txt +sed -i '/future==/d' requirements/py3_common_lock.txt + +deactivate diff --git a/version.py b/version.py index a61a642ea2..29b1ffc20b 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ ZULIP_VERSION = "1.6.0+git" -PROVISION_VERSION = '5.13' +PROVISION_VERSION = '5.14'