memcached: Switch from pylibmc to python-binary-memcached.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

analytics/migrations/0015_clear_duplicate_counts.py

@@ -10,7 +10,7 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor)
     The backstory is that Django's unique_together indexes do not properly
     handle the subgroup=None corner case (allowing duplicate rows that have a
     subgroup of None), which meant that in race conditions, rather than updating
-    an existing row for the property/realm/time with subgroup=None, Django would
+    an existing row for the property/(realm, stream, user)/time with subgroup=None, Django would
     create a duplicate row.
 
     In the next migration, we'll add a proper constraint to fix this bug, but
@@ -20,26 +20,32 @@ def clear_duplicate_counts(apps: StateApps, schema_editor: DatabaseSchemaEditor)
     this means deleting the extra rows, but for LoggingCountStat objects, we need to
     additionally combine the sums.
     """
-    RealmCount = apps.get_model('analytics', 'RealmCount')
+    count_tables = dict(realm=apps.get_model('analytics', 'RealmCount'),
+                        user=apps.get_model('analytics', 'UserCount'),
+                        stream=apps.get_model('analytics', 'StreamCount'),
+                        installation=apps.get_model('analytics', 'InstallationCount'))
 
-    realm_counts = RealmCount.objects.filter(subgroup=None).values(
+    for name, count_table in count_tables.items():
-        'realm_id', 'property', 'end_time').annotate(
+        value = [name, 'property', 'end_time']
+        if name == 'installation':
+            value = ['property', 'end_time']
+        counts = count_table.objects.filter(subgroup=None).values(*value).annotate(
             Count('id'), Sum('value')).filter(id__count__gt=1)
 
-    for realm_count in realm_counts:
+        for count in counts:
-        realm_count.pop('id__count')
+            count.pop('id__count')
-        total_value = realm_count.pop('value__sum')
+            total_value = count.pop('value__sum')
-        duplicate_counts = list(RealmCount.objects.filter(**realm_count))
+            duplicate_counts = list(count_table.objects.filter(**count))
-        first_count = duplicate_counts[0]
+            first_count = duplicate_counts[0]
-        if realm_count['property'] in ["invites_sent::day", "active_users_log:is_bot:day"]:
+            if count['property'] in ["invites_sent::day", "active_users_log:is_bot:day"]:
-            # For LoggingCountStat objects, the right fix is to combine the totals;
+                # For LoggingCountStat objects, the right fix is to combine the totals;
-            # for other CountStat objects, we expect the duplicates to have the same value.
+                # for other CountStat objects, we expect the duplicates to have the same value.
-            # And so all we need to do is delete them.
+                # And so all we need to do is delete them.
-            first_count.value = total_value
+                first_count.value = total_value
-            first_count.save()
+                first_count.save()
-        to_cleanup = duplicate_counts[1:]
+            to_cleanup = duplicate_counts[1:]
-        for duplicate_count in to_cleanup:
+            for duplicate_count in to_cleanup:
-            duplicate_count.delete()
+                duplicate_count.delete()
 
 class Migration(migrations.Migration):
 

docs/development/setup-advanced.md

@@ -177,7 +177,7 @@ git remote add -f upstream https://github.com/zulip/zulip.git
 
 ```
 doas pkg_add sudo bash gcc postgresql-server redis rabbitmq \
-    memcached libmemcached py-Pillow py-cryptography py-cffi
+    memcached py-Pillow py-cryptography py-cffi
 
 # Point environment to custom include locations and use newer GCC
 # (needed for Node modules):

docs/overview/changelog.md

@@ -7,6 +7,26 @@ All notable changes to the Zulip server are documented in this file.
 This section lists notable unreleased changes; it is generally updated
 in bursts.
 
+### 3.1 -- July 30, 2020
+
+- Removed unused `short_name` field from the User model.  This field
+  had no purpose and could leak the local part of email addresses
+  when email address visiblity was restricted.
+- Fixed a bug where loading spinners would sometimes not be displayed.
+- Fixed incoming email gateway exception with unstructured headers.
+- Fixed AlertWords not being included in data import/export.
+- Fixed Twitter previews not including a clear link to the tweet.
+- Fixed compose box incorrectly opening after uploading a file in a
+  message edit widget.
+- Fixed exception in SAML integration with encrypted assertions.
+- Fixed an analytics migration bug that could cause upgrading from 2.x
+  releases to fail.
+- Added a Thinkst Canary integration (and renamed the old one, which
+  was actually an integration for canarytokens.org).
+- Reformatted the frontend codebase using prettier.  This change was
+  included in this maintenance release to ensure backporting patches
+  from master remains easy.
+
 ### 3.0 -- July 16, 2020
 
 #### Highlights

docs/production/upgrade-or-modify.md

@@ -339,11 +339,11 @@ working correctly.
     ```
     apt remove upstart -y
     /home/zulip/deployments/current/scripts/zulip-puppet-apply -f
-    pg_dropcluster 9.5 main --stop
+    pg_dropcluster 11 main --stop
     systemctl stop postgresql
-    pg_upgradecluster -m upgrade 9.3 main
+    pg_upgradecluster -m upgrade 9.6 main
-    pg_dropcluster 9.3 main
+    pg_dropcluster 9.6 main
-    apt remove postgresql-9.3
+    apt remove postgresql-9.6
     systemctl start postgresql
     service memcached restart
     ```

puppet/zulip/manifests/app_frontend_base.pp

@@ -151,15 +151,4 @@ class zulip::app_frontend_base {
     mode    => '0755',
     source  => 'puppet:///modules/zulip/nagios_plugins/zulip_app_frontend',
   }
-
-  if $::osfamily == 'debian' {
-    # The pylibmc wheel looks for SASL plugins in the wrong place.
-    file { '/usr/lib64':
-      ensure => directory,
-    }
-    file { '/usr/lib64/sasl2':
-      ensure => link,
-      target => "/usr/lib/${::rubyplatform}/sasl2",
-    }
-  }
 }

requirements/common.in

@@ -79,10 +79,11 @@ pika
 psycopg2 --no-binary psycopg2
 
 # Needed for memcached usage
-pylibmc
+# https://github.com/jaysonsantos/python-binary-memcached/pull/230, https://github.com/jaysonsantos/python-binary-memcached/pull/231
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git
 
-# Needed for compression support in memcached via pylibmc
+# Needed for compression support in memcached via python-binary-memcached
-django-pylibmc
+django-bmemcached
 
 # Needed for zerver/tests/test_timestamp.py
 python-dateutil

requirements/dev.txt

@@ -273,6 +273,9 @@ django-bitfield==2.0.1 \
     --hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
     --hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
     # via -r requirements/common.in
+django-bmemcached==0.3.0 \
+    --hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
+    # via -r requirements/common.in
 django-cookies-samesite==0.6.6 \
     --hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
     # via -r requirements/common.in
@@ -288,10 +291,6 @@ django-phonenumber-field==3.0.1 \
     --hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
     --hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
     # via django-two-factor-auth
-django-pylibmc==0.6.1 \
-    --hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
-    --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
-    # via -r requirements/common.in
 django-sendfile2==0.6.0 \
     --hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
     # via -r requirements/common.in
@@ -775,13 +774,6 @@ pyjwt==1.7.1 \
     --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
     --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
     # via -r requirements/common.in, apns2, social-auth-core, twilio
-pylibmc==1.6.1 \
-    --hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
-    --hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
-    --hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
-    --hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
-    --hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
-    # via -r requirements/common.in, django-pylibmc
 pyoembed==0.1.2 \
     --hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
     # via -r requirements/common.in
@@ -796,6 +788,9 @@ pyparsing==2.4.7 \
 pyrsistent==0.16.0 \
     --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
     # via jsonschema
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
+    --hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
+    # via -r requirements/common.in, django-bmemcached
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@@ -925,7 +920,9 @@ sh==1.12.14 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
+    --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
+    --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
+    # via argon2-cffi, automat, aws-sam-translator, cfn-lint, cryptography, django-bitfield, docker, ecdsa, hypchat, isodate, jsonschema, junit-xml, libthumbor, moto, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging, parsel, pip-tools, protego, pyopenssl, python-binary-memcached, python-dateutil, python-debian, python-jose, qrcode, responses, social-auth-app-django, social-auth-core, talon, traitlets, twilio, w3lib, websocket-client, zulip
 snakeviz==2.1.0 \
     --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
     --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
@@ -1115,6 +1112,9 @@ ua-parser==0.10.0 \
     --hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
     --hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
     # via django-cookies-samesite
+uhashring==1.2 \
+    --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
+    # via python-binary-memcached
 https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
     --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
     # via -r requirements/common.in

requirements/docs.txt

@@ -106,6 +106,8 @@ requests==2.24.0 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
+    --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
+    --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
     # via packaging
 snowballstemmer==2.0.0 \
     --hash=sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0 \

requirements/prod.txt

@@ -185,6 +185,9 @@ django-bitfield==2.0.1 \
     --hash=sha256:83bfa27da718caff436f646369ce58e2d9f922e1f3d65a93f0b731a835cbfc58 \
     --hash=sha256:ab340eb50cdb1e8c005594b9f8170a95a698102d06cf3f5031763be2750a8862 \
     # via -r requirements/common.in
+django-bmemcached==0.3.0 \
+    --hash=sha256:4e4b7d97216dbae331c1de10e699ca22804b94ec3a90d2762dd5d146e6986a8a \
+    # via -r requirements/common.in
 django-cookies-samesite==0.6.6 \
     --hash=sha256:a26dc27bfc446279c981a301b053eff845b93d9ba62798e281c90584a7ccaa4a \
     # via -r requirements/common.in
@@ -200,10 +203,6 @@ django-phonenumber-field==3.0.1 \
     --hash=sha256:1ab19f723928582fed412bd9844221fa4ff466276d8526b8b4a9913ee1487c5e \
     --hash=sha256:794ebbc3068a7af75aa72a80cb0cec67e714ff8409a965968040f1fd210b2d97 \
     # via django-two-factor-auth
-django-pylibmc==0.6.1 \
-    --hash=sha256:02b591933a029eb552388cced713028f3c6cbb021639fc8de388bd1ca87981d4 \
-    --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 \
-    # via -r requirements/common.in
 django-sendfile2==0.6.0 \
     --hash=sha256:7f850040ddc29c9c42192ed85b915465a3ed7cced916c4fafdd5eda057dd06ec \
     # via -r requirements/common.in
@@ -554,13 +553,6 @@ pyjwt==1.7.1 \
     --hash=sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e \
     --hash=sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96 \
     # via -r requirements/common.in, apns2, social-auth-core, twilio
-pylibmc==1.6.1 \
-    --hash=sha256:01a7e2e3fa9fcd7a791c7818a80a07e7a381aee988a5d810a1c1e6f7a9a288fd \
-    --hash=sha256:6fff384e3c30af029bbac87f88b3fab14ae87b50103d389341d9b3e633349a3f \
-    --hash=sha256:8a8dd406487d419d58c6d944efd91e8189b360a0c4d9e8c6ebe3990d646ae7e9 \
-    --hash=sha256:c749b4251c1137837d00542b62992b96cd2aed639877407f66291120dd6de2ff \
-    --hash=sha256:e6c0c452336db0868d0de521d48872c2a359b1233b974c6b32c36ce68abc4820 \
-    # via -r requirements/common.in, django-pylibmc
 pyoembed==0.1.2 \
     --hash=sha256:0f755c8308039f1e49238e95ea94ef16aa08add9f32075ba13ab9b65f32ff582 \
     # via -r requirements/common.in
@@ -571,6 +563,9 @@ pyopenssl==19.1.0 \
 pyrsistent==0.16.0 \
     --hash=sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3 \
     # via jsonschema
+https://github.com/jaysonsantos/python-binary-memcached/archive/364ce723ea73290a6ae27551cab28070424fd280.zip#egg=python-binary-memcached==0.29.0+git \
+    --hash=sha256:0e6f4c7c34c71e29e1daa53cca6598dcbdb8bd49d7c6aaac6c02d93bdc2d5a8f \
+    # via -r requirements/common.in, django-bmemcached
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
@@ -661,7 +656,9 @@ s3transfer==0.3.3 \
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
+    --hash=sha256:8ce375b18ae4a749516d7e6c6fbbf8be6177c53974f53534d8eadb646cd279b1 \
+    --hash=sha256:92ad876fb6a201a7e23a6b85ea96d9643a51e285667c253a8653643804f7cb68 \
+    # via argon2-cffi, cryptography, django-bitfield, hypchat, isodate, jsonschema, libthumbor, openapi-core, openapi-schema-validator, openapi-spec-validator, pyopenssl, python-binary-memcached, python-dateutil, qrcode, social-auth-app-django, social-auth-core, talon, traitlets, twilio, zulip
 social-auth-app-django==4.0.0 \
     --hash=sha256:2c69e57df0b30c9c1823519c5f1992cbe4f3f98fdc7d95c840e091a752708840 \
     --hash=sha256:567ad0e028311541d7dfed51d3bf2c60440a6fd236d5d4d06c5a618b3d6c57c5 \
@@ -751,6 +748,9 @@ ua-parser==0.10.0 \
     --hash=sha256:46ab2e383c01dbd2ab284991b87d624a26a08f72da4d7d413f5bfab8b9036f8a \
     --hash=sha256:47b1782ed130d890018d983fac37c2a80799d9e0b9c532e734c67cf70f185033 \
     # via django-cookies-samesite
+uhashring==1.2 \
+    --hash=sha256:f7304ca2ff763bbf1e2f8a78f21131721811619c5841de4f8c98063344906931 \
+    # via python-binary-memcached
 https://github.com/zulip/ultrajson/archive/70ac02becc3e11174cd5072650f885b30daab8a8.zip#egg=ujson==1.35+git \
     --hash=sha256:e95c20f47093dc7376ddf70b95489979375fb6e88b8d7e4b5576d917dda8ef5a \
     # via -r requirements/common.in

scripts/lib/setup-yum-repo

@@ -14,7 +14,6 @@ done
 
 is_centos=false
 is_rhel=false
-is_rhel_registered=false
 if [ -e /etc/centos-release ]; then
     is_centos=true
     yum install -y epel-release
@@ -27,12 +26,6 @@ if [ -e /etc/centos-release ]; then
 elif grep -q "Red Hat" /etc/redhat-release; then
     is_rhel=true
     yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    if subscription-manager status; then
-        # See https://access.redhat.com/discussions/2217891#comment-1032701
-        is_rhel_registered=true
-        # libmemcached-devel can be installed directly if the machine is registered
-        subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms"
-    fi
 fi
 
 yum update -y
@@ -51,10 +44,6 @@ if [ "$is_centos" = true ]; then
     # https://pgroonga.github.io/install/centos.html
     yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
 elif [ "$is_rhel" = true ]; then
-    if [ "$is_rhel_registered" = false ]; then
-        echo "This machine is unregistered; installing libmemcached-devel from a CentOS mirror ..."
-        yum localinstall -y http://mirror.centos.org/centos/7/os/x86_64/Packages/libmemcached-devel-1.0.16-5.el7.x86_64.rpm
-    fi
     yum localinstall -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-latest-x86_64/pgdg-redhat10-10-2.noarch.rpm
     yum localinstall -y https://packages.groonga.org/centos/groonga-release-latest.noarch.rpm
 else

scripts/lib/setup_venv.py

@@ -17,7 +17,6 @@ VENV_DEPENDENCIES = [
     "zlib1g-dev",             # Needed to handle compressed PNGs with Pillow
     "libjpeg-dev",          # Needed to handle JPEGs with Pillow
     "libldap2-dev",
-    "libmemcached-dev",
     "python3-dev",          # Needed to install typed-ast dependency of mypy
     "python3-pip",
     "virtualenv",
@@ -43,7 +42,6 @@ COMMON_YUM_VENV_DEPENDENCIES = [
     "zlib-devel",
     "libjpeg-turbo-devel",
     "openldap-devel",
-    "libmemcached-devel",
     # Needed by python-xmlsec:
     "gcc"
     "python3-devel",

scripts/setup/flush-memcached

@@ -9,15 +9,10 @@ from scripts.lib.setup_path import setup_path
 
 setup_path()
 
-import pylibmc
+import bmemcached
 
 from zproject import settings
 
-assert isinstance(settings.CACHES["default"], dict)  # for mypy
+cache = settings.CACHES["default"]
-pylibmc.Client(
+assert isinstance(cache, dict)  # for mypy
-    [settings.MEMCACHED_LOCATION],
+bmemcached.Client((cache["LOCATION"],), **cache["OPTIONS"]).flush_all()
-    binary=True,
-    username=settings.MEMCACHED_USERNAME,
-    password=settings.MEMCACHED_PASSWORD,
-    behaviors=settings.CACHES["default"]["OPTIONS"],
-).flush_all()

static/js/upload.js

@@ -228,7 +228,7 @@ exports.setup_upload = function (config) {
         }
         const split_uri = uri.split("/");
         const filename = split_uri[split_uri.length - 1];
-        if (!compose_state.composing()) {
+        if (config.mode === "compose" && !compose_state.composing()) {
             compose_actions.start("stream");
         }
         const absolute_uri = exports.make_upload_absolute(uri);

static/styles/zulip.scss

@@ -2133,8 +2133,6 @@ div.topic_edit_spinner .loading_indicator_spinner {
 }
 
 #do_delete_message_spinner {
-    display: none;
-    width: 0;
     margin: 0 auto;
 }
 

tools/ci/Dockerfile.template

@@ -92,7 +92,7 @@ RUN apt-get update \
     memcached rabbitmq-server redis-server \
     hunspell-en-us supervisor libssl-dev puppet \
     gettext libffi-dev libfreetype6-dev zlib1g-dev \
-    libjpeg-dev libldap2-dev libmemcached-dev \
+    libjpeg-dev libldap2-dev \
     libxml2-dev libxslt1-dev libpq-dev moreutils \
     {extra_packages}
 

tools/test-install/prepare-base

@@ -52,7 +52,7 @@ run apt-get install -y --no-install-recommends \
   memcached redis-server \
   hunspell-en-us supervisor libssl-dev puppet \
   gettext libffi-dev libfreetype6-dev zlib1g-dev libjpeg-dev \
-  libldap2-dev libmemcached-dev \
+  libldap2-dev \
   libxml2-dev libxslt1-dev libpq-dev \
   virtualenv \
   "${extra_packages[@]}"

version.py

@@ -1,6 +1,6 @@
 import os
 
-ZULIP_VERSION = "4.0-dev+git"
+ZULIP_VERSION = "3.1"
 # Add information on number of commits and commit hash to version, if available
 zulip_git_version_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'zulip-git-version')
 if os.path.exists(zulip_git_version_file):
@@ -10,7 +10,7 @@ if os.path.exists(zulip_git_version_file):
             ZULIP_VERSION = version
 
 LATEST_MAJOR_VERSION = "3.0"
-LATEST_RELEASE_VERSION = "3.0"
+LATEST_RELEASE_VERSION = "3.1"
 LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2020/07/16/zulip-3-0-released/"
 LATEST_DESKTOP_VERSION = "5.3.0"
 
@@ -44,4 +44,4 @@ API_FEATURE_LEVEL = 27
 #   historical commits sharing the same major version, in which case a
 #   minor version bump suffices.
 
-PROVISION_VERSION = '90.1'
+PROVISION_VERSION = '91.0'

zerver/lib/email_mirror.py

@@ -1,5 +1,6 @@
 import logging
 import re
+from email.headerregistry import AddressHeader
 from email.message import EmailMessage
 from typing import Dict, List, Optional, Tuple
 
@@ -310,9 +311,14 @@ def find_emailgateway_recipient(message: EmailMessage) -> str:
 
     for header_name in recipient_headers:
         for header_value in message.get_all(header_name, []):
-            for addr in header_value.addresses:
+            if isinstance(header_value, AddressHeader):
-                if match_email_re.match(addr.addr_spec):
+                emails = [addr.addr_spec for addr in header_value.addresses]
-                    return addr.addr_spec
+            else:
+                emails = [str(header_value)]
+
+            for email in emails:
+                if match_email_re.match(email):
+                    return email
 
     raise ZulipEmailForwardError("Missing recipient in mirror email")
 

zerver/tests/test_email_mirror.py

@@ -232,6 +232,36 @@ class TestStreamEmailMessagesSuccess(ZulipTestCase):
         self.assertEqual(get_display_recipient(message.recipient), stream.name)
         self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
 
+    # Test receiving an email with the address on an UnstructuredHeader
+    # (e.g. Envelope-To) instead of an AddressHeader (e.g. To).
+    # https://github.com/zulip/zulip/issues/15864
+    def test_receive_stream_email_messages_other_header_success(self) -> None:
+        user_profile = self.example_user('hamlet')
+        self.login_user(user_profile)
+        self.subscribe(user_profile, "Denmark")
+        stream = get_stream("Denmark", user_profile.realm)
+
+        stream_to_address = encode_email_address(stream)
+
+        incoming_valid_message = EmailMessage()
+        incoming_valid_message.set_content('TestStreamEmailMessages Body')
+
+        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
+        incoming_valid_message['From'] = self.example_email('hamlet')
+        # Simulate a mailing list
+        incoming_valid_message['To'] = "foo-mailinglist@example.com"
+        incoming_valid_message['Envelope-To'] = stream_to_address
+        incoming_valid_message['Reply-to'] = self.example_email('othello')
+
+        process_message(incoming_valid_message)
+
+        # Hamlet is subscribed to this stream so should see the email message from Othello.
+        message = most_recent_message(user_profile)
+
+        self.assertEqual(message.content, "TestStreamEmailMessages Body")
+        self.assertEqual(get_display_recipient(message.recipient), stream.name)
+        self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
+
     def test_receive_stream_email_messages_blank_subject_success(self) -> None:
         user_profile = self.example_user('hamlet')
         self.login_user(user_profile)

zilencer/management/commands/populate_db.py

@@ -4,7 +4,7 @@ import random
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple
 
-import pylibmc
+import bmemcached
 import ujson
 from django.conf import settings
 from django.contrib.sessions.models import Session
@@ -79,13 +79,9 @@ def clear_database() -> None:
     # With `zproject.test_settings`, we aren't using real memcached
     # and; we only need to flush memcached if we're populating a
     # database that would be used with it (i.e. zproject.dev_settings).
-    if default_cache['BACKEND'] == 'django_pylibmc.memcached.PyLibMCCache':
+    if default_cache['BACKEND'] == 'django_bmemcached.memcached.BMemcached':
-        pylibmc.Client(
+        bmemcached.Client(
-            [default_cache['LOCATION']],
+            (default_cache['LOCATION'],), **default_cache['OPTIONS'],
-            binary=True,
-            username=default_cache["USERNAME"],
-            password=default_cache["PASSWORD"],
-            behaviors=default_cache["OPTIONS"],
         ).flush_all()
 
     model: Any = None  # Hack because mypy doesn't know these are model classes

zproject/backends.py

@@ -38,6 +38,7 @@ from jwt.exceptions import PyJWTError
 from lxml.etree import XMLSyntaxError
 from onelogin.saml2.errors import OneLogin_Saml2_Error
 from onelogin.saml2.response import OneLogin_Saml2_Response
+from onelogin.saml2.settings import OneLogin_Saml2_Settings
 from requests import HTTPError
 from social_core.backends.apple import AppleIdAuth
 from social_core.backends.azuread import AzureADOAuth2
@@ -1774,8 +1775,7 @@ class SAMLAuthBackend(SocialAuthMixin, SAMLAuth):
 
         return data
 
-    @classmethod
+    def get_issuing_idp(self, SAMLResponse: str) -> Optional[str]:
-    def get_issuing_idp(cls, SAMLResponse: str) -> Optional[str]:
         """
         Given a SAMLResponse, returns which of the configured IdPs is declared as the issuer.
         This value MUST NOT be trusted as the true issuer!
@@ -1786,11 +1786,12 @@ class SAMLAuthBackend(SocialAuthMixin, SAMLAuth):
         of the configured IdPs' information to use for parsing and validating the response.
         """
         try:
-            resp = OneLogin_Saml2_Response(settings={}, response=SAMLResponse)
+            config = self.generate_saml_config()
+            saml_settings = OneLogin_Saml2_Settings(config, sp_validation_only=True)
+            resp = OneLogin_Saml2_Response(settings=saml_settings, response=SAMLResponse)
             issuers = resp.get_issuers()
-        except cls.SAMLRESPONSE_PARSING_EXCEPTIONS:
+        except self.SAMLRESPONSE_PARSING_EXCEPTIONS:
-            logger = logging.getLogger(f"zulip.auth.{cls.name}")
+            self.logger.info("Error while parsing SAMLResponse:", exc_info=True)
-            logger.info("Error while parsing SAMLResponse:", exc_info=True)
             return None
 
         for idp_name, idp_config in settings.SOCIAL_AUTH_SAML_ENABLED_IDPS.items():

zproject/computed_settings.py

@@ -326,24 +326,17 @@ RABBITMQ_PASSWORD = get_secret("rabbitmq_password")
 
 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
 
-# Compress large values being stored in memcached; this is important
-# for at least the realm_users cache.
-PYLIBMC_MIN_COMPRESS_LEN = 100 * 1024
-PYLIBMC_COMPRESS_LEVEL = 1
-
 MEMCACHED_PASSWORD = get_secret("memcached_password")
 
 CACHES = {
     'default': {
-        'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
+        'BACKEND': 'django_bmemcached.memcached.BMemcached',
         'LOCATION': MEMCACHED_LOCATION,
-        'TIMEOUT': 3600,
-        'BINARY': True,
-        'USERNAME': MEMCACHED_USERNAME,
-        'PASSWORD': MEMCACHED_PASSWORD,
         'OPTIONS': {
-            'tcp_nodelay': True,
+            'socket_timeout': 3600,
-            'retry_timeout': 1,
+            'username': MEMCACHED_USERNAME,
+            'password': MEMCACHED_PASSWORD,
+            'pickle_protocol': 4,
         },
     },
     'database': {