camo: Add endpoint to handle camo requests.

This endpoint serves requests which might originate from an image
preview link which had an http url and the message holding the image
link was rendered before we introduced thumbnailing. In that case
we would have used a camo proxy to proxy http content over https and
avoid mix content warnings.

In near future, we plan to drop use of camo and just rely on thumbor
to serve such images. This endpoint helps maintain backward
compatibility for links which were already rendered.

docs/subsystems/thumbnailing.md

@@ -16,7 +16,9 @@ Thumbor is responsible for a few things in Zulip:
 
 * Serving all image content over HTTPS, even if the original/upstream
   image was hosted on HTTP (this was previously done by `camo` in
-  older versions of Zulip).  This is important to avoid mixed-content
+  older versions of Zulip; the `THUMBOR_SERVES_CAMO` setting controls
+  whether Thumbor will serve the old-style Camo URLs that might be
+  present in old messages).  This is important to avoid mixed-content
   warnings from browsers (which look very bad), and does have some
   real security benefit in protecting our users from malicious
   content.

zerver/lib/camo.py

@@ -17,3 +17,8 @@ def get_camo_url(url: str) -> str:
     if settings.CAMO_URI == '':
         return url
     return "%s%s" % (settings.CAMO_URI, generate_camo_url(url))
+
+def is_camo_url_valid(digest: str, url: str) -> bool:
+    camo_url = generate_camo_url(url)
+    camo_url_digest = camo_url.split('/')[0]
+    return camo_url_digest == digest

zerver/lib/thumbnail.py

@@ -30,7 +30,9 @@ def get_source_type(url: str) -> str:
         return THUMBOR_LOCAL_FILE_TYPE
     return THUMBOR_S3_TYPE
 
-def generate_thumbnail_url(path: str, size: str='0x0') -> str:
+def generate_thumbnail_url(path: str,
+                           size: str='0x0',
+                           is_camo_url: bool=False) -> str:
     if not (path.startswith('https://') or path.startswith('http://')):
         path = '/' + path
 
@@ -48,14 +50,18 @@ def generate_thumbnail_url(path: str, size: str='0x0') -> str:
     width, height = map(int, size.split('x'))
     crypto = CryptoURL(key=settings.THUMBOR_KEY)
 
+    smart_crop_enabled = True
     apply_filters = ['no_upscale()']
+    if is_camo_url:
+        smart_crop_enabled = False
+        apply_filters.append('quality(100)')
     if size != '0x0':
         apply_filters.append('sharpen(0.5,0.2,true)')
 
     encrypted_url = crypto.generate(
         width=width,
         height=height,
-        smart=True,
+        smart=smart_crop_enabled,
         filters=apply_filters,
         image_url=image_url
     )

zerver/tests/test_camo.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from zerver.lib.test_classes import ZulipTestCase
+
+class CamoURLTest(ZulipTestCase):
+    def test_legacy_camo_url(self) -> None:
+        # Test with valid hex and url pair
+        result = self.client_get("/external_content/0f50f0bda30b6e65e9442c83ddb4076c74e75f96/687474703a2f2f7777772e72616e646f6d2e736974652f696d616765732f666f6f6261722e6a706567")
+        self.assertEqual(result.status_code, 302, result)
+        self.assertIn('/filters:no_upscale():quality(100)/aHR0cDovL3d3dy5yYW5kb20uc2l0ZS9pbWFnZXMvZm9vYmFyLmpwZWc=/source_type/external', result.url)
+
+        # Test with invalid hex and url pair
+        result = self.client_get("/external_content/074c5e6c9c6d4ce97db1c740d79dc561cf7eb379/687474703a2f2f7777772e72616e646f6d2e736974652f696d616765732f666f6f6261722e6a706567")
+        self.assertEqual(result.status_code, 403, result)
+        self.assert_in_response("Not a valid URL.", result)
+
+    def test_with_thumbor_disabled(self) -> None:
+        with self.settings(THUMBOR_SERVES_CAMO=False):
+            result = self.client_get("/external_content/074c5e6c9c6d4ce97db1c740d79dc561cf7eb379/687474703a2f2f7777772e72616e646f6d2e736974652f696d616765732f666f6f6261722e6a706567")
+            self.assertEqual(result.status_code, 404, result)

zerver/views/camo.py

@@ -0,0 +1,23 @@
+from django.conf import settings
+from django.shortcuts import redirect
+from django.utils.translation import ugettext as _
+from django.http import (
+    HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseNotFound
+)
+from zerver.lib.camo import is_camo_url_valid
+from zerver.lib.thumbnail import generate_thumbnail_url
+
+import codecs
+
+def handle_camo_url(request: HttpRequest, digest: str,
+                    received_url: str) -> HttpResponse:
+    if not settings.THUMBOR_SERVES_CAMO:
+        return HttpResponseNotFound()
+
+    hex_encoded_url = received_url.encode('utf-8')
+    hex_decoded_url = codecs.decode(hex_encoded_url, 'hex')
+    original_url = hex_decoded_url.decode('utf-8')  # type: ignore # https://github.com/python/typeshed/issues/300
+    if is_camo_url_valid(digest, original_url):
+        return redirect(generate_thumbnail_url(original_url, is_camo_url=True))
+    else:
+        return HttpResponseForbidden(_("<p>Not a valid URL.</p>"))

zproject/settings.py

@@ -209,6 +209,7 @@ DEFAULT_SETTINGS = {
     'REMOTE_POSTGRES_HOST': '',
     'REMOTE_POSTGRES_SSLMODE': '',
     'THUMBOR_URL': '',
+    'THUMBOR_SERVES_CAMO': False,
     'THUMBNAIL_IMAGES': False,
     'SENDFILE_BACKEND': None,
 

zproject/test_settings.py

@@ -160,6 +160,7 @@ SLOW_QUERY_LOGS_STREAM = None
 
 THUMBOR_URL = 'http://127.0.0.1:9995'
 THUMBNAIL_IMAGES = True
+THUMBOR_SERVES_CAMO = True
 
 # Logging the emails while running the tests adds them
 # to /emails page.

zproject/urls.py

@@ -20,6 +20,7 @@ import zerver.tornado.views
 import zerver.views
 import zerver.views.auth
 import zerver.views.archive
+import zerver.views.camo
 import zerver.views.compatibility
 import zerver.views.home
 import zerver.views.email_mirror
@@ -585,6 +586,14 @@ urls += [
 urls += url(r'^report/csp_violations$', zerver.views.report.report_csp_violations,
             name='zerver.views.report.report_csp_violations'),
 
+# This url serves as a way to provide backward compatibility to messages
+# rendered at the time Zulip used camo for doing http -> https conversion for
+# such links with images previews. Now thumbor can be used for serving such
+# images.
+urls += url(r'^external_content/(?P<digest>[\S]+)/(?P<received_url>[\S]+)',
+            zerver.views.camo.handle_camo_url,
+            name='zerver.views.camo.handle_camo_url'),
+
 # Incoming webhook URLs
 # We don't create urls for particular git integrations here
 # because of generic one below