Release Zulip Server 2.0.6.

This allows the system to get updates to the Groonga repository
signing key, so `apt update` doesn’t start failing when the key
changes (like it recently did).

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

docs/conf.py

@@ -52,7 +52,7 @@ author = 'The Zulip Team'
 # The short X.Y version.
 version = '2.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.4'
+release = '2.0.6'
 
 # This allows us to insert a warning that appears only on an unreleased
 # version, e.g. to say that something is likely to have changed.

docs/overview/changelog.md

@@ -7,6 +7,21 @@ All notable changes to the Zulip server are documented in this file.
 This section lists notable unreleased changes; it is generally updated
 in bursts.
 
+### 2.0.6 -- 2019-09-23
+
+- Updated signing keys for the PGroonga repository for Debian Stretch.
+- Fixed creation of linkifiers with URLs containing &.
+- Fixed a subtle bug that could cause the message list to suddenly
+  scroll up in certain rare race conditions.
+
+### 2.0.5 -- 2019-09-11
+
+- CVE-2019-16215: Fix DoS vulnerability in Markdown LINK_RE.
+- CVE-2019-16216: Fix MIME type validation.
+- Fixed email gateway postfix configuration for Ubuntu Bionic.
+- Fixed support for hidden_by_limit messages in Slack import.
+- Fixed confusing output from the `knight` management command.
+
 ### 2.0.4 -- 2019-06-29
 
 - Fixed several configuration-dependent bugs that caused

docs/production/install.md

@@ -1,14 +1,16 @@
 # Production Installation
 
-Make sure you want to install a Zulip production server. If you'd
-instead like to test or develop a new feature, we recommend the
-[Zulip development server](../development/overview.html#requirements) instead.
-If you just want to play around with Zulip and see what it looks like, you
-can create a test organization at <https://zulipchat.com>.
-
 You'll need an Ubuntu or Debian system that satisfies
-[the installation requirements](../production/requirements.html), or
-you can use Zulip's [experimental Docker image](../production/deployment.html#zulip-in-docker).
+[the installation requirements](../production/requirements.html). Alternatively,
+you can use a preconfigured
+[Digital Ocean droplet](https://marketplace.digitalocean.com/apps/zulip), or
+Zulip's
+[experimental Docker image](../production/deployment.html#zulip-in-docker).
+
+Note that if you're developing for Zulip, you should install Zulip's
+[development environment](../development/overview.html) instead. If
+you're just looking to play around with Zulip and see what it looks like,
+you can create a test organization at <https://zulipchat.com/new>.
 
 ## Step 1: Download the latest release
 

frontend_tests/node_tests/message_list.js

@@ -182,7 +182,7 @@ run_test('message_range', () => {
 
 run_test('updates', () => {
     var list = new MessageList({});
-    list.view.rerender_the_whole_thing = noop;
+    list.view.rerender_preserving_scrolltop = noop;
 
     var messages = [
         {

puppet/zulip/templates/postfix/main.cf.erb

@@ -15,6 +15,7 @@ smtpd_use_tls=yes
 smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 
+smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated reject_unauth_destination
 myhostname = <%= @fqdn %>
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases

scripts/lib/install

@@ -12,6 +12,7 @@ Other options:
   --self-signed-cert
   --no-init-db
   --cacert
+  --no-dist-upgrade
 
 The --hostname and --email options are required,
 unless --no-init-db is set and --certbot is not.
@@ -21,7 +22,7 @@ EOF
 
 # Shell option parsing.  Over time, we'll want to move some of the
 # environment variables below into this self-documenting system.
-args="$(getopt -o '' --long help,no-init-db,self-signed-cert,certbot,hostname:,email:,cacert: -n "$0" -- "$@")"
+args="$(getopt -o '' --long help,no-init-db,no-dist-upgrade,self-signed-cert,certbot,hostname:,email:,cacert: -n "$0" -- "$@")"
 eval "set -- $args"
 while true; do
     case "$1" in

scripts/lib/setup-apt-repo

@@ -65,4 +65,8 @@ else
     apt-get update && rm -f "$STAMP_FILE"
 fi
 
+if [ "$release" = "stretch" ]; then
+    apt-get install -y groonga-keyring
+fi
+
 echo "$DEPENDENCIES_HASH" > "$DEPENDENCIES_HASH_FILE"

scripts/setup/pgroonga-debian.asc

@@ -12,19 +12,80 @@ rIbTnCy/oJRrEDbhTDhjhbMZgskWEVl7LguxW5y2WL/snj8E7bRBZ3Jvb25nYSBL
 ZXkgKGdyb29uZ2EgT2ZmaWNpYWwgU2lnbmluZyBLZXkpIDxwYWNrYWdlc0Bncm9v
 bmdhLm9yZz6IYgQTEQIAIgUCT5uUPwIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC
 F4AACgkQcqdJa0VJlCnCeQCeNNMnOiri+zdLBU3EmBBuZZFet44AmgPwGZHfgA1r
-SrzymknxZI07SFIsiEYEEBECAAYFAk+qcWgACgkQF0I/ZByDfzFXJwCg2ZeJ4+7i
-KLSjio53xauxgjXfdL8An2wATGnD/z0Xm2iIqqHhcRvYBoaeiQIcBBABAgAGBQJP
-qnGpAAoJEJHRj88Hn4AHKO0P+wZGCLiB7GVR16z0spaHrgFvQKn8bVNYmfonwYn0
-9uOD0UdB3AivV55STrmv48nCz8BvPUE0P9DLmU0+a7Rdz5aYdkDGKqQJkx/uc1jf
-3p9b+ikbx8qgUSZK5TUsilZcFpTgsEDZAJtdc5k2QQ6C6rYe8DD8pXPRgfmgqsaI
-frb8Xdg80c8K8XOZR3FQC/sEQdRiWpxNBgWXTgX6PAHo7Ci83p1hW4guYlFegSoY
-1+pwLGaC9ALhRjXwMMsjpAqTmgbkBtQww3iW/ysN5uOlyoPJG3utJTjOyRkH0SgT
-QR2amwTkMEApF2bOcZyz5cSlZPRMcpEAp/p2zR96LQs3CM6UiYOoACRk+BT7yvQY
-Q3mMK302fcIeEeq3sa1x4m6u9SL5YzX134zgsDOdcXYD9D+/lsJV+fvDa/A1CFxm
-zaMZB7/4e13CIfUL2u9toHZBcxG8LRkOckXO2BTuBr2Y1u0tIgDemoQeIXwoiTeB
-guMSN8gT+8BppkGwCSoab1XUv4E31gRzE0EU2ShZyIj7XdQ1qKk+yrVBULhpexRF
-FjgyO9xzO4omxrtB6huFLbDNRhqqKeEVC0PyXBFyXC4n2hgVunz/UkiWIPiKpYEM
-epcLCIBVS9H7Eg4nD8ejpTFpwrv3uJsKErQOhDNim30sueqhRAqEIm/7RQUK2o8H
-0fbF
-=mkRj
+SrzymknxZI07SFIssAIAA4hGBBARAgAGBQJPqnFoAAoJEBdCP2Qcg38xVycAoNmX
+iePu4ii0o4qOd8WrsYI133S/AJ9sAExpw/89F5toiKqh4XEb2AaGnrACAACJAhwE
+EAECAAYFAk+qcakACgkQkdGPzwefgAco7Q/7BkYIuIHsZVHXrPSyloeuAW9Aqfxt
+U1iZ+ifBifT244PRR0HcCK9XnlJOua/jycLPwG89QTQ/0MuZTT5rtF3Plph2QMYq
+pAmTH+5zWN/en1v6KRvHyqBRJkrlNSyKVlwWlOCwQNkAm11zmTZBDoLqth7wMPyl
+c9GB+aCqxoh+tvxd2DzRzwrxc5lHcVAL+wRB1GJanE0GBZdOBfo8AejsKLzenWFb
+iC5iUV6BKhjX6nAsZoL0AuFGNfAwyyOkCpOaBuQG1DDDeJb/Kw3m46XKg8kbe60l
+OM7JGQfRKBNBHZqbBOQwQCkXZs5xnLPlxKVk9ExykQCn+nbNH3otCzcIzpSJg6gA
+JGT4FPvK9BhDeYwrfTZ9wh4R6rexrXHibq71IvljNfXfjOCwM51xdgP0P7+WwlX5
++8Nr8DUIXGbNoxkHv/h7XcIh9Qva722gdkFzEbwtGQ5yRc7YFO4GvZjW7S0iAN6a
+hB4hfCiJN4GC4xI3yBP7wGmmQbAJKhpvVdS/gTfWBHMTQRTZKFnIiPtd1DWoqT7K
+tUFQuGl7FEUWODI73HM7iibGu0HqG4UtsM1GGqop4RULQ/JcEXJcLifaGBW6fP9S
+SJYg+IqlgQx6lwsIgFVL0fsSDicPx6OlMWnCu/e4mwoStA6EM2KbfSy56qFECoQi
+b/tFBQrajwfR9sWwAgAAiQIcBBABAgAGBQJPvHGKAAoJENIsGog0VdRICFsP/j9Q
+enxWaiMAfLQOaC2wpLW4BrEmdkhbs4qSeAfFwof/jO7vehmYkda6RHHVtE5xN6UQ
+tTFUuLqLwNaMdz6sgBi1jc/02oYcajxLJENwAk3o3GaSfadd6HeMLKrqyf8rA1eY
+Bs3/6F2MEpPMCvfZTddMFPyGfhstjvgzxBUoxbW7sCqj2kEci14azVFhf8jijStF
+EFQVr1eh3oAaJjlOi5/uGB+H3yz8kRONBFyvLaBRSLepI3/5rU2wC0ItvlCISvdf
+PCsOF7A8ho9N+cSpqym7zGA3u6kValmrLz/w7BRgbPX52MCh0ULBmarge0U+X6c1
+D0Z/o3wxt5a0EApW8FN3WJK1vhV+cyTjAqJIO+B3c8hDfr7C7/4fcSSHdzdiPsg9
+TgVIq36Q7Fl/cqR4hx2QGNr7ErZMzXLXuMK7ZFqQ9hqDBmS8r/E0z+ze53BTG52q
+W5jjxdtc6l+KB29FnE5K+8EmuiR9dVbdrhV5DlrYNjiQG/pAtq8NdHh+yd3Q8mME
+yd45shAZQM9LdiAW0AmNCjUCQzTbHbSHbaoE7V5qZcHznRNJ00l2zCGuJeW9aTjp
+7gN3E+jtp54s03EGaxahwxIatI9bGKCxHPDF3zwGCweOh7ywYWap+7bF9WlOOgOL
++IKjX9Jn7c8RC5PzWHQLR4941zCQWMiHeFQvxnLKsAIAAJkCDQRabqOPARAAv+FK
+JmXGbdsIw2+FqBRsVcQyEmn+JP8ZYkAs91ddQhzedyH93RrKozkKyU0abuXrlxKH
+nG9GIolFiNvHg9SGo067rpxg4yOu9v7t/okehmtcJO54mv+bZaOCzGJeb2vwUJMV
+SMfeTnKBwYeOpaFQJ22qvjsn1fq/XSCyVH9bcQCeuUSBmUopIJKEgqFZ0cDYYS6O
+LLLuuTOqOJUbCOnVD/MKZ8vHSvBKUZUsK24nK5ZfpQQ8RWSIdTipZdruame5rUCj
+jGwsFYYpXch63VBtTyOMpyCA7/f/K/ln+MHqAqpZ7CnSq3h3/fdMlnXtKkjE2Z8o
+yX8mPKSjT5M9nFhyqcYis+g2g0lyQP7KAAJp3kYd+9C9PqvWG1C/0ymt4gZjWTmo
+icUvsNqbeJ+2dBj6HM+ejLRh2NMy6ZZq5v/s/GwR/lb9TA0BLdYSBPn3QdrRiAwo
+I73oBFUXJAnVm+mR3kb3JhO/1SqGNBmWQttPPABMnOh3fegdDVhA6aKAxU35GmeS
+FXNVVtyuxRepdF5vvXy85i+y/L8wJzGQ+cQdQJw8P1qIsex5e7k3VVwZRVnJIrlN
+jfN5rU/yH5bCt6AX2e1FSSVs14mNYhohA5gKpxDZkojfgzShmbrCTO+awnFDOO1y
+MQtbVfm2G1X/06D3Zl35+RJUfZzI5e+42PkxJqkAEQEAAbRBR3Jvb25nYSBLZXkg
+KEdyb29uZ2EgT2ZmaWNpYWwgU2lnbmluZyBLZXkpIDxwYWNrYWdlc0Bncm9vbmdh
+Lm9yZz6JAk4EEwEIADgWIQQnAfMXz8zLl1yt6cJiTPd0NIOSJQUCWm6jjwIbAwUL
+CQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRBiTPd0NIOSJb+SD/9CvmBD9OdRB/Nk
+f/Rdiq4XOPnCP5La66i5NXb7IZKVOZuY3PqfGJ43XPTq6qBKLSlCAwkFbclXZP2z
+lne+bYAknHJbJbFy3aceoRmAOnkSgFeMj7V0J0nfBbrUForvaaGjDzIDgx/G3/D7
+gxq5C4zYXQNGbi39XommyDveB2hdaNbWdI1YOlG2fDMC7S4VPaMfNVYxDzrLBokf
+DAgYB2QsYX3toEkdZpsQxPXfNCm9g2G+JApHtjI6EpLtEodgbSJf/Rp9lRqeRNZJ
+X/lUY6MSujI73n2O8huZklgbBPGJioaJc1Q+EFEl1mNHt3nfCz7d/FlzON3lCbdb
+g+X3fl6FX00fmUmlNj/XBG2G/InnNt7dgH1x6MIaTnFKsi0p0xj3U3lZmUJqei0o
+7SAGMM+QY3at5A5D2YpGChmMpGLqDuICLjogQqFhClIpfm0yBt3lNa3wi41rFr0y
+X5rMawfYV/g4y0FDT+xh/wxgMcLeKErtIYZRq6QLNYq9PSniEK3gssPsj/LdshcE
+OC6NuuENXrgJdvQ2rgfI3wM0uFVUelNPE6cbc680waSoCNMGPl4Nk4ExtSa3WRQG
+H4J6aZHkX8L6aWedRCoLTLQ2LB49Ow7Ol3KE66XfU7Y+JBfznvGPMYOzZEZjxdwl
+ytR0bHU5H6b0dpVBDCgojAVphRlpyLACAAOIXQQQEQgAHRYhBMl+RkmiBR0M6hpz
++XKnSWtFSZQpBQJabssTAAoJEHKnSWtFSZQprgsAoKAistI/y3CMRyarH1va32fj
+HNiyAJ9i7XDqQTBbrT1yMhISjrE9lscr3LACAAO5Ag0EWm6jjwEQALUgeK8Dm49G
+cenJZS6WOPBFDfxMZM24d4NbRcbJyGJ9RJoky4CQTY42QbAJ4V3bC/p9kD9hW/w6
+aeDB6G8EuV4NQJL0A1dy5AD0N4fVmYDTfbNze0DzJSrs1eCwhExYDGgvcR36tlrm
+K9ZMvzq85ej1mQ/g8iUPgPPkpFI5BtgPHKlHghyDmK/JYcFwLmjz0R1cbsKPWEXM
+AdESq5UojBH51xnXMKwB3aUHyU2gO7iqQxf5p1lXPMOX4ssnMuiz3DGWUkaQhy1s
+gJZtzYiX3To1NTijbdSKlRJ+CXlG/vFagexgDWfMYrjbcmLQllKNt+Sc0FPUcUqG
+tXKFxSI0ny3tUv+eTF8d5kqSaJk67nfHB6abxLjmXLoUMwMtlnyKPxHpKXJEe7+U
+lTDo8UGT8ZILscftAWZPW2WLn8FOCXQQcIDG/G/7Adi374u/WQpOhi5Y86MPnbLV
+53Bx2pP+XTbYDKFc/2wfKOVpxLHDDWCIsM3DL7YY63B1pSt5B3lJUnvjxuHhXtWN
+G9UIgkaFCzvY8jgAXIVBnWAJcdkruUCwOD0tXmK45YPyK7fNLte4kWcOLxoh7rND
+qo/wUyK9pB5G/CdE8J1G+T+egF+6qUM9IEIR8OJWQt9uR00ogMXdCAVztm4ZImX6
+boiW5SbWEzHnzv4mdH0WlbQtzjrKkwClABEBAAGJAjYEGAEIACAWIQQnAfMXz8zL
+l1yt6cJiTPd0NIOSJQUCWm6jjwIbDAAKCRBiTPd0NIOSJXtkD/9IthyF6CnQBazM
+QO/JIHp3Kfe/9ll+4hSSSc9tpijYznXpNQVv3rQQwVooL3oV5XoATK8H6kv2IOyh
+tGq2szMt9YQ2JuGdjcOb5Mc2A+QWD3Tn7KCcwpIdOWiL74EWBKX6yM5JG103nI0X
+y1W5FSyCJ6lB1xDoCKgUdqrgfEwAgkt8kDeoi57j9wYilt2d5+UK85pXqNgOMKxR
+0tLCHcngN1XKq4irfjBVVlh205qjsTApVzLrYYe0nGae/yejmGwCLMu37yd/XiNf
+jMi56gEYvIU/ZehqJQf00O4Cmneggu5A+KCG7cEULtuPLcwUho7swdsm+bTCNAAM
+CvhSFeTUAs1atIOIsw1rStonPPOvjd0Ig3qWyaVs2PgK8xh21aLg5tIXmn2bTegc
+mFJGGfv6YAkkPAKqtjJ/RPVZZH93PzouR590dAX/mZWZYRfo6ipxgv6ALhL51z7l
+E/Zqcdg7TSkG2tY2NJnoXLROpXg9Bs7gFkb6ia7YeSJTz3Q0uBbQMqWkQyrj1RB8
+i18m28J9/OkLiSryhsyLh4UULqm7PUNyNKMV31TaIBQVvutKtLZ/GLWmPc/tBSa3
+Uy5CBG5oTrh1xo/3ZO0JRUW2CYU+gMvTRowmLP2uhU7JOAtz3QAerpHpNhtRdfPs
+0HAz6RfxSr0qk9eQec/UPhOATDujkLACAAM=
+=IBdD
 -----END PGP PUBLIC KEY BLOCK-----

static/js/message_list.js

@@ -52,7 +52,7 @@ exports.MessageList.prototype = {
         var render_info;
 
         if (interior_messages.length > 0) {
-            self.view.rerender_the_whole_thing();
+            self.view.rerender_preserving_scrolltop(true);
             return true;
         }
         if (top_messages.length > 0) {

static/js/message_list_view.js

@@ -1037,7 +1037,7 @@ MessageListView.prototype = {
         return true;
     },
 
-    rerender_preserving_scrolltop: function () {
+    rerender_preserving_scrolltop: function (discard_rendering_state) {
         // old_offset is the number of pixels between the top of the
         // viewable window and the selected message
         var old_offset;
@@ -1046,6 +1046,13 @@ MessageListView.prototype = {
         if (selected_in_view) {
             old_offset = selected_row.offset().top;
         }
+        if (discard_rendering_state) {
+            // If we know that the existing render is invalid way
+            // (typically because messages appear out-of-order), then
+            // we discard the message_list rendering state entirely.
+            this.clear_rendering_state(true);
+            this.update_render_window(this.list.selected_idx(), false);
+        }
         return this.rerender_with_target_scrolltop(selected_row, old_offset);
     },
 
@@ -1224,13 +1231,6 @@ MessageListView.prototype = {
         this.maybe_rerender();
     },
 
-    rerender_the_whole_thing: function () {
-        // TODO: Figure out if we can unify this with this.list.rerender().
-        this.clear_rendering_state(true);
-        this.update_render_window(this.list.selected_idx(), false);
-        this.render(this.list.all_messages().slice(this._render_win_start, this._render_win_end), 'bottom');
-    },
-
     clear_table: function () {
         // We do not want to call .empty() because that also clears
         // jQuery data.  This does mean, however, that we need to be

templates/zerver/accounts_home.html

@@ -97,6 +97,17 @@ $(function () {
                             </form>
                         </div>
                         {% endif %}
+
+                        {% if azuread_auth_enabled %}
+                        <div class="login-social">
+                            <form class="form-inline azure-wrapper" action="{{ url('signup-social', args=('azuread-oauth2',)) }}" method="get">
+                                <input type='hidden' name='multiuse_object_key' value='{{ multiuse_object_key }}' />
+                                <button class="login-social-button full-width">
+                                    {{ _('Sign up with %(identity_provider)s', identity_provider="Azure AD") }}
+                                </button>
+                            </form>
+                        </div>
+                        {% endif %}
                     {% endif %}
                 </div>
             </div>

version.py

@@ -1,6 +1,6 @@
-ZULIP_VERSION = "2.0.4"
+ZULIP_VERSION = "2.0.6"
 LATEST_MAJOR_VERSION = "2.0"
-LATEST_RELEASE_VERSION = "2.0.4"
+LATEST_RELEASE_VERSION = "2.0.6"
 LATEST_RELEASE_ANNOUNCEMENT = "https://blog.zulip.org/2019/03/01/zulip-2-0-released/"
 
 # Bump the minor PROVISION_VERSION to indicate that folks should provision

zerver/data_import/slack.py

@@ -732,10 +732,11 @@ def process_message_files(message: ZerverFieldsT,
     markdown_links = []
 
     for fileinfo in files:
-        if fileinfo.get('mode', '') == 'tombstone':
+        if fileinfo.get('mode', '') in ['tombstone', 'hidden_by_limit']:
             # Slack sometimes includes tombstone mode files with no
             # real data on the actual file (presumably in cases where
-            # the file was deleted).
+            # the file was deleted). hidden_by_limit mode is for files
+            # that are hidden because of 10k cap in free plan.
             continue
 
         url = fileinfo['url_private']

zerver/lib/bugdown/__init__.py

@@ -1484,7 +1484,7 @@ def get_link_re() -> str:
 
     # [text](url) or [text](<url>) or [text](url "title")
     LINK_RE = NOIMG + BRK + \
-        r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)'''
+        r'''\(\s*(<(?:[^<>\\]|\\.)*>|(\([^()]*\)|[^()])*?)\s*(('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")\s*)?\)'''
     return normal_compile(LINK_RE)
 
 def prepare_realm_pattern(source: str) -> str:

zerver/lib/upload.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, Tuple
+from typing import Optional, Tuple
 
 from django.utils.translation import ugettext as _
 from django.conf import settings
@@ -41,6 +41,17 @@ DEFAULT_EMOJI_SIZE = 64
 MAX_EMOJI_GIF_SIZE = 128
 MAX_EMOJI_GIF_FILE_SIZE_BYTES = 128 * 1024 * 1024  # 128 kb
 
+INLINE_MIME_TYPES = [
+    "application/pdf",
+    "image/gif",
+    "image/jpeg",
+    "image/png",
+    "image/webp",
+    # To avoid cross-site scripting attacks, DO NOT add types such
+    # as application/xhtml+xml, application/x-shockwave-flash,
+    # image/svg+xml, text/html, or text/xml.
+]
+
 # Performance Note:
 #
 # For writing files to S3, the file could either be stored in RAM
@@ -267,10 +278,11 @@ def upload_image_to_s3(
     key.set_metadata("user_profile_id", str(user_profile.id))
     key.set_metadata("realm_id", str(user_profile.realm_id))
 
+    headers = {}
     if content_type is not None:
-        headers = {'Content-Type': content_type}  # type: Optional[Dict[str, str]]
-    else:
-        headers = None
+        headers["Content-Type"] = content_type
+    if content_type not in INLINE_MIME_TYPES:
+        headers["Content-Disposition"] = "attachment"
 
     key.set_contents_from_string(contents, headers=headers)  # type: ignore # https://github.com/python/typeshed/issues/1552
 

zerver/management/commands/knight.py

@@ -38,25 +38,27 @@ ONLY perform this on customer request from an authorized person.
         email = options['email']
         realm = self.get_realm(options)
 
-        profile = self.get_user(email, realm)
+        user = self.get_user(email, realm)
 
         if options['grant']:
-            if profile.has_perm(options['permission'], profile.realm):
+            if (user.is_realm_admin and options['permission'] == "administer" or
+                    user.is_api_super_user and options['permission'] == "api_super_user"):
                 raise CommandError("User already has permission for this realm.")
             else:
                 if options['ack']:
-                    do_change_is_admin(profile, True, permission=options['permission'])
+                    do_change_is_admin(user, True, permission=options['permission'])
                     print("Done!")
                 else:
                     print("Would have granted %s %s rights for %s" % (
-                          email, options['permission'], profile.realm.string_id))
+                          email, options['permission'], user.realm.string_id))
         else:
-            if profile.has_perm(options['permission'], profile.realm):
+            if (user.is_realm_admin and options['permission'] == "administer" or
+                    user.is_api_super_user and options['permission'] == "api_super_user"):
                 if options['ack']:
-                    do_change_is_admin(profile, False, permission=options['permission'])
+                    do_change_is_admin(user, False, permission=options['permission'])
                     print("Done!")
                 else:
                     print("Would have removed %s's %s rights on %s" % (email, options['permission'],
-                                                                       profile.realm.string_id))
+                                                                       user.realm.string_id))
             else:
                 raise CommandError("User did not have permission for this realm!")

zerver/models.py

@@ -594,7 +594,7 @@ def filter_pattern_validator(value: str) -> None:
         raise ValidationError(error_msg)
 
 def filter_format_validator(value: str) -> None:
-    regex = re.compile(r'^([\.\/:a-zA-Z0-9#_?=-]+%\(([a-zA-Z0-9_-]+)\)s)+[a-zA-Z0-9_-]*$')
+    regex = re.compile(r'^([\.\/:a-zA-Z0-9#_?=&-]+%\(([a-zA-Z0-9_-]+)\)s)+[/a-zA-Z0-9#_?=&-]*$')
 
     if not regex.match(value):
         raise ValidationError(_('Invalid URL format string.'))

zerver/tests/test_realm_filters.py

@@ -58,25 +58,25 @@ class RealmFilterTest(ZulipTestCase):
         self.assertIsNotNone(re.match(data['pattern'], 'ZUL2-15'))
 
         data['pattern'] = r'_code=(?P<id>[0-9a-zA-Z]+)'
-        data['url_format_string'] = 'https://realm.com/my_realm_filter/?value=%(id)s'
+        data['url_format_string'] = 'https://example.com/product/%(id)s/details'
         result = self.client_post("/json/realm/filters", info=data)
         self.assert_json_success(result)
         self.assertIsNotNone(re.match(data['pattern'], '_code=123abcdZ'))
 
         data['pattern'] = r'PR (?P<id>[0-9]+)'
-        data['url_format_string'] = 'https://realm.com/my_realm_filter/?value=%(id)s'
+        data['url_format_string'] = 'https://example.com/web#view_type=type&model=model&action=12345&id=%(id)s'
         result = self.client_post("/json/realm/filters", info=data)
         self.assert_json_success(result)
         self.assertIsNotNone(re.match(data['pattern'], 'PR 123'))
 
         data['pattern'] = r'lp/(?P<id>[0-9]+)'
-        data['url_format_string'] = 'https://realm.com/my_realm_filter/?value=%(id)s'
+        data['url_format_string'] = 'https://realm.com/my_realm_filter/?value=%(id)s&sort=reverse'
         result = self.client_post("/json/realm/filters", info=data)
         self.assert_json_success(result)
         self.assertIsNotNone(re.match(data['pattern'], 'lp/123'))
 
         data['pattern'] = r'lp:(?P<id>[0-9]+)'
-        data['url_format_string'] = 'https://realm.com/my_realm_filter/?value=%(id)s'
+        data['url_format_string'] = 'https://realm.com/my_realm_filter/?sort=reverse&value=%(id)s'
         result = self.client_post("/json/realm/filters", info=data)
         self.assert_json_success(result)
         self.assertIsNotNone(re.match(data['pattern'], 'lp:123'))

zerver/views/upload.py

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext as _
 
 from zerver.lib.response import json_success, json_error
 from zerver.lib.upload import upload_message_image_from_request, get_local_file_path, \
-    get_signed_upload_url, check_upload_within_quota
+    get_signed_upload_url, check_upload_within_quota, INLINE_MIME_TYPES
 from zerver.models import UserProfile, validate_attachment_request
 from django.conf import settings
 from sendfile import sendfile
@@ -38,13 +38,11 @@ def serve_local(request: HttpRequest, path_id: str) -> HttpResponse:
     # consistent format (urlquoted).  For more details on filename*
     # and filename, see the below docs:
     # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
-    attachment = True
-    file_type = guess_type(local_path)[0]
-    if file_type is not None and (file_type.startswith("image/") or
-                                  file_type == "application/pdf"):
-        attachment = False
+    mimetype, encoding = guess_type(local_path)
+    attachment = mimetype not in INLINE_MIME_TYPES
 
-    return sendfile(request, local_path, attachment=attachment)
+    return sendfile(request, local_path, attachment=attachment,
+                    mimetype=mimetype, encoding=encoding)
 
 def serve_file_backend(request: HttpRequest, user_profile: UserProfile,
                        realm_id_str: str, filename: str) -> HttpResponse: