realm-emoji: Add realm emoji uploading instead url providing.

- Add file_name field to `RealmEmoji` model and migration.
- Add emoji upload supporting to Upload backends.
- Add uploaded file processing to emoji views.
- Use emoji source url as based for display url.
- Change emoji form for image uploading.
- Fix back-end tests.
- Fix front-end tests.
- Add tests for emoji uploading.

Fixes #1134

frontend_tests/casper_tests/10-admin.js

@@ -71,7 +71,7 @@ casper.then(function () {
     casper.waitUntilVisible('.admin-emoji-form', function () {
         casper.fill('form.admin-emoji-form', {
             name: 'MouseFace',
-            url: 'http://zulipdev.com:9991/static/images/integrations/logos/jenkins.png',
+            emoji_file_input: 'static/images/logo/zulip-icon-128x128.png',
         }, true);
     });
 });
@@ -85,7 +85,7 @@ casper.then(function () {
 casper.then(function () {
     casper.waitUntilVisible('.emoji_row', function () {
         casper.test.assertSelectorHasText('.emoji_row .emoji_name', 'MouseFace');
-        casper.test.assertExists('.emoji_row img[src="http://zulipdev.com:9991/static/images/integrations/logos/jenkins.png"]');
+        casper.test.assertExists('.emoji_row img[src="/user_avatars/1/emoji/MouseFace.png"]');
         casper.click('.emoji_row button.delete');
     });
 });

static/js/emoji.js

@@ -41,8 +41,8 @@ exports.update_emojis = function update_emojis(realm_emojis) {
     // Copy the default emoji list and add realm-specific emoji to it
     exports.emojis = default_emojis.slice(0);
     _.each(realm_emojis, function (data, name) {
-        exports.emojis.push({emoji_name: name, emoji_url: data.display_url, is_realm_emoji: true});
+        exports.emojis.push({emoji_name: name, emoji_url: data.source_url, is_realm_emoji: true});
-        exports.realm_emojis[name] = {emoji_name: name, emoji_url: data.display_url};
+        exports.realm_emojis[name] = {emoji_name: name, emoji_url: data.source_url};
     });
     // Add the Zulip emoji to the realm emojis list
     exports.emojis.push(zulip_emoji);
@@ -70,6 +70,26 @@ exports.initialize = function initialize() {
 
 exports.update_emojis(page_params.realm_emoji);
 
+exports.build_emoji_upload_widget = function () {
+
+    var get_file_input = function () {
+        return $('#emoji_file_input');
+    };
+
+    var file_name_field = $('#emoji-file-name');
+    var input_error = $('#emoji_file_input_error');
+    var clear_button = $('#emoji_image_clear_button');
+    var upload_button = $('#emoji_upload_button');
+
+    return upload_widget.build_widget(
+        get_file_input,
+        file_name_field,
+        input_error,
+        clear_button,
+        upload_button
+    );
+};
+
 return exports;
 }());
 if (typeof module !== 'undefined') {

static/js/settings_emoji.js

@@ -21,7 +21,7 @@ exports.populate_emoji = function (emoji_data) {
         emoji_table.append(templates.render('admin_emoji_list', {
             emoji: {
                 name: name, source_url: data.source_url,
-                display_url: data.display_url,
+                display_url: data.source_url,
                 author: data.author,
                 is_admin: page_params.is_admin,
             },
@@ -61,28 +61,38 @@ exports.set_up = function () {
         });
     });
 
+    var emoji_widget = emoji.build_emoji_upload_widget();
+
     $(".organization").on("submit", "form.admin-emoji-form", function (e) {
         e.preventDefault();
         e.stopPropagation();
         var emoji_status = $('#admin-emoji-status');
         var emoji = {};
+        var formData = new FormData();
         _.each($(this).serializeArray(), function (obj) {
             emoji[obj.name] = obj.value;
         });
-
+        $.each($('#emoji_file_input')[0].files, function (i, file) {
+            formData.append('file-' + i, file);
+        });
         channel.put({
             url: "/json/realm/emoji/" + encodeURIComponent(emoji.name),
-            data: $(this).serialize(),
+            data: formData,
+            cache: false,
+            processData: false,
+            contentType: false,
             success: function () {
                 $('#admin-emoji-status').hide();
                 ui_report.success(i18n.t("Custom emoji added!"), emoji_status);
                 $("form.admin-emoji-form input[type='text']").val("");
+                emoji_widget.clear();
             },
             error: function (xhr) {
                 $('#admin-emoji-status').hide();
                 var errors = JSON.parse(xhr.responseText).msg;
                 xhr.responseText = JSON.stringify({msg: errors});
                 ui_report.error(i18n.t("Failed!"), xhr, emoji_status);
+                emoji_widget.clear();
             },
         });
     });

static/templates/admin_emoji_list.handlebars

@@ -4,7 +4,7 @@
         <span class="emoji_name">{{name}}</span>
     </td>
     <td>
-        <span class="emoji_image"><a href="{{source_url}}"><img src="{{display_url}}" alt="{{name}}" /></a></span>
+        <span class="emoji_image"><a href="{{source_url}}"><img src="{{source_url}}" alt="{{name}}" /></a></span>
     </td>
     <td>
         {{#if author}}

static/templates/settings/emoji-settings-admin.handlebars

@@ -23,8 +23,12 @@
                     <input type="text" name="name" id="emoji_name" placeholder="mouse_face" />
                 </div>
                 <div class="input-group">
-                    <label for="emoji_url">{{t "Emoji URL" }}</label>
+                    <p id="emoji-file-name"></p>
-                    <input type="text" name="url" id="emoji_url" placeholder="http://emojipedia-us.s3.amazonaws.com/cache/46/7f/467fe69069c408e07517621f263ea9b5.png" />
+                    <input type="file" name="emoji_file_input" class="notvisible"
+                 id="emoji_file_input" value="{{t 'Upload emoji' }}"/>
+                    <button class="btn btn-default display-none" id="emoji_image_clear_button">{{t "Clear emoji image" }}</button>
+                    <button class="button white" id="emoji_upload_button">{{t "Upload emoji" }}</button>
+                    <p id="emoji_file_input_error" class="text-error"></p>
                 </div>
                 <button type="submit" class="button white rounded sea-green">
                     {{t 'Add emoji' }}

zerver/lib/actions.py

@@ -3095,9 +3095,9 @@ def notify_realm_emoji(realm):
                  realm_emoji=realm.get_emoji())
     send_event(event, active_user_ids(realm))
 
-def check_add_realm_emoji(realm, name, img_url, author=None):
+def check_add_realm_emoji(realm, name, file_name, author=None):
     # type: (Realm, Text, Text, Optional[UserProfile]) -> None
-    emoji = RealmEmoji(realm=realm, name=name, img_url=img_url, author=author)
+    emoji = RealmEmoji(realm=realm, name=name, file_name=file_name, author=author)
     emoji.full_clean()
     emoji.save()
     notify_realm_emoji(realm)

zerver/lib/bugdown/__init__.py

@@ -727,7 +727,7 @@ class Emoji(markdown.inlinepatterns.Pattern):
             realm_emoji = db_data['emoji']
 
         if current_message and name in realm_emoji:
-            return make_realm_emoji(realm_emoji[name]['display_url'], orig_syntax)
+            return make_realm_emoji(realm_emoji[name]['source_url'], orig_syntax)
         elif name == 'zulip':
             return make_realm_emoji('/static/generated/emoji/images/emoji/unicode/zulip.png', orig_syntax)
         elif name in name_to_codepoint:

zerver/lib/emoji.py

@@ -1,12 +1,13 @@
 from __future__ import absolute_import
 
+import os
 import re
 
 from django.utils.translation import ugettext as _
 from typing import Text
-
 from zerver.lib.bugdown import name_to_codepoint
 from zerver.lib.request import JsonableError
+from zerver.lib.upload import upload_backend
 from zerver.models import Realm, UserProfile
 
 def check_valid_emoji(realm, emoji_name):
@@ -29,3 +30,13 @@ def check_valid_emoji_name(emoji_name):
     if re.match('^[0-9a-zA-Z.\-_]+(?<![.\-_])$', emoji_name):
         return
     raise JsonableError(_("Invalid characters in emoji name"))
+
+def get_emoji_url(emoji_file_name, realm_id):
+    # type: (Text, int) -> Text
+    return upload_backend.get_emoji_url(emoji_file_name, realm_id)
+
+
+def get_emoji_file_name(emoji_file_name, emoji_name):
+    # type: (Text, Text) -> Text
+    _, image_ext = os.path.splitext(emoji_file_name)
+    return ''.join((emoji_name, image_ext))

zerver/lib/upload.py

@@ -20,9 +20,9 @@ from boto.s3.connection import S3Connection
 from mimetypes import guess_type, guess_extension
 
 from zerver.lib.str_utils import force_bytes, force_str
-from zerver.models import get_user_profile_by_email, get_user_profile_by_id
+from zerver.models import get_user_profile_by_email, get_user_profile_by_id, RealmEmoji
 from zerver.models import Attachment
-from zerver.models import Realm, UserProfile, Message
+from zerver.models import Realm, RealmEmoji, UserProfile, Message
 
 from six.moves import urllib
 import base64
@@ -36,6 +36,7 @@ import logging
 
 DEFAULT_AVATAR_SIZE = 100
 MEDIUM_AVATAR_SIZE = 500
+DEFAULT_EMOJI_SIZE = 64
 
 # Performance Note:
 #
@@ -99,6 +100,26 @@ def resize_avatar(image_data, size=DEFAULT_AVATAR_SIZE):
     im.save(out, format='png')
     return out.getvalue()
 
+
+def resize_emoji(image_data, size=DEFAULT_EMOJI_SIZE):
+    # type: (binary_type, int) -> binary_type
+    try:
+        im = Image.open(io.BytesIO(image_data))
+        image_format = im.format
+        if image_format == 'GIF' and im.is_animated:
+            if im.size[0] > size or im.size[1] > size:
+                raise JsonableError(
+                    _("Animated emoji can't be larger than 64px in width or height."))
+            else:
+                return image_data
+        im = ImageOps.fit(im, (size, size), Image.ANTIALIAS)
+    except IOError:
+        raise BadImageError("Could not decode image; did you upload an image file?")
+    out = io.BytesIO()
+    im.save(out, format=image_format)
+    return out.getvalue()
+
+
 ### Common
 
 class ZulipUploadBackend(object):
@@ -131,6 +152,14 @@ class ZulipUploadBackend(object):
         # type: (int, int) -> Text
         raise NotImplementedError()
 
+    def upload_emoji_image(self, emoji_file, emoji_file_name, user_profile):
+        # type: (File, Text, UserProfile) -> None
+        raise NotImplementedError()
+
+    def get_emoji_url(self, emoji_file_name, realm_id):
+        # type: (Text, int) -> Text
+        raise NotImplementedError()
+
 
 ### S3
 
@@ -221,7 +250,9 @@ def get_realm_for_filename(path):
         return None
     return get_user_profile_by_id(key.metadata["user_profile_id"]).realm_id
 
+
 class S3UploadBackend(ZulipUploadBackend):
+
     def upload_message_image(self, uploaded_file_name, uploaded_file_size,
                              content_type, file_data, user_profile, target_realm=None):
         # type: (Text, int, Optional[Text], binary_type, UserProfile, Optional[Realm]) -> Text
@@ -357,6 +388,40 @@ class S3UploadBackend(ZulipUploadBackend):
             resized_medium
         )
 
+    def upload_emoji_image(self, emoji_file, emoji_file_name, user_profile):
+        # type: (File, Text, UserProfile) -> None
+        content_type = guess_type(emoji_file.name)[0]
+        bucket_name = settings.S3_AVATAR_BUCKET
+        emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+            realm_id=user_profile.realm_id,
+            emoji_file_name=emoji_file_name
+        )
+
+        image_data = emoji_file.read()
+        resized_image_data = resize_emoji(image_data)
+        upload_image_to_s3(
+            bucket_name,
+            ".".join((emoji_path, "original")),
+            content_type,
+            user_profile,
+            image_data,
+        )
+        upload_image_to_s3(
+            bucket_name,
+            emoji_path,
+            content_type,
+            user_profile,
+            resized_image_data,
+        )
+
+    def get_emoji_url(self, emoji_file_name, realm_id):
+        # type: (Text, int) -> Text
+        bucket = settings.S3_AVATAR_BUCKET
+        emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id,
+                                                        emoji_file_name=emoji_file_name)
+        return u"https://%s.s3.amazonaws.com/%s" % (bucket, emoji_path)
+
+
 ### Local
 
 def mkdirs(path):
@@ -459,6 +524,30 @@ class LocalUploadBackend(ZulipUploadBackend):
         resized_medium = resize_avatar(image_data, MEDIUM_AVATAR_SIZE)
         write_local_file('avatars', file_path + '-medium.png', resized_medium)
 
+    def upload_emoji_image(self, emoji_file, emoji_file_name, user_profile):
+        # type: (File, Text, UserProfile) -> None
+        emoji_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+            realm_id= user_profile.realm_id,
+            emoji_file_name=emoji_file_name
+        )
+
+        image_data = emoji_file.read()
+        resized_image_data = resize_emoji(image_data)
+        write_local_file(
+            'avatars',
+            ".".join((emoji_path, "original")),
+            image_data)
+        write_local_file(
+            'avatars',
+            emoji_path,
+            resized_image_data)
+
+    def get_emoji_url(self, emoji_file_name, realm_id):
+        # type: (Text, int) -> Text
+        return os.path.join(
+            u"/user_avatars",
+            RealmEmoji.PATH_ID_TEMPLATE.format(realm_id=realm_id, emoji_file_name=emoji_file_name))
+
 # Common and wrappers
 if settings.LOCAL_UPLOADS_DIR is not None:
     upload_backend = LocalUploadBackend() # type: ZulipUploadBackend
@@ -477,6 +566,10 @@ def upload_icon_image(user_file, user_profile):
     # type: (File, UserProfile) -> None
     upload_backend.upload_realm_icon_image(user_file, user_profile)
 
+def upload_emoji_image(emoji_file, emoji_file_name, user_profile):
+    # type: (File, Text, UserProfile) -> None
+    upload_backend.upload_emoji_image(emoji_file, emoji_file_name, user_profile)
+
 def upload_message_image(uploaded_file_name, uploaded_file_size,
                          content_type, file_data, user_profile, target_realm=None):
     # type: (Text, int, Optional[Text], binary_type, UserProfile, Optional[Realm]) -> Text

zerver/migrations/0077_add_file_name_field_to_realm_emoji.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-03-09 05:23
+from __future__ import unicode_literals
+
+import os
+import io
+import logging
+import requests
+
+from mimetypes import guess_type
+
+from PIL import Image
+from PIL import ImageOps
+from django.db import migrations, models
+
+from django.db import migrations
+from django.db.backends.postgresql_psycopg2.schema import DatabaseSchemaEditor
+from django.db.migrations.state import StateApps
+from django.conf import settings
+from boto.s3.key import Key
+from boto.s3.connection import S3Connection
+from requests import ConnectionError, Response
+from typing import Dict, Text, Tuple, Optional, Union
+
+from six import binary_type
+
+
+def force_str(s, encoding='utf-8'):
+    # type: (Union[Text, binary_type], Text) -> str
+    """converts a string to a native string"""
+    if isinstance(s, str):
+        return s
+    elif isinstance(s, Text):
+        return s.encode(str(encoding))
+    elif isinstance(s, binary_type):
+        return s.decode(encoding)
+    else:
+        raise TypeError("force_str expects a string type")
+
+
+class Uploader(object):
+    def __init__(self):
+        # type: () -> None
+        self.path_template = "{realm_id}/emoji/{emoji_file_name}"
+        self.emoji_size = (64, 64)
+
+    def upload_files(self, response, resized_image, dst_path_id):
+        # type: (Response, binary_type, Text) -> None
+        raise NotImplementedError()
+
+    def get_dst_path_id(self, realm_id, url, emoji_name):
+        # type: (int, Text, Text) -> Tuple[Text,Text]
+        _, image_ext = os.path.splitext(url)
+        file_name = ''.join((emoji_name, image_ext))
+        return file_name, self.path_template.format(realm_id=realm_id, emoji_file_name=file_name)
+
+    def resize_emoji(self, image_data):
+        # type: (binary_type) -> Optional[binary_type]
+        im = Image.open(io.BytesIO(image_data))
+        format_ = im.format
+        if format_ == 'GIF' and im.is_animated:
+            return None
+        im = ImageOps.fit(im, self.emoji_size, Image.ANTIALIAS)
+        out = io.BytesIO()
+        im.save(out, format_)
+        return out.getvalue()
+
+    def upload_emoji(self, realm_id, image_url, emoji_name):
+        # type: (int, Text, Text) -> Optional[Text]
+        file_name, dst_path_id = self.get_dst_path_id(realm_id, image_url, emoji_name)
+        try:
+            response = requests.get(image_url, stream=True)
+        except ConnectionError:
+            return None
+        if response.status_code != 200:
+            return None
+        try:
+            resized_image = self.resize_emoji(response.content)
+        except IOError:
+            return None
+        self.upload_files(response, resized_image, dst_path_id)
+        return file_name
+
+
+class LocalUploader(Uploader):
+    def __init__(self):
+        # type: () -> None
+        super(LocalUploader, self).__init__()
+
+    @staticmethod
+    def mkdirs(path):
+        # type: (Text) -> None
+        dirname = os.path.dirname(path)
+        if not os.path.isdir(dirname):
+            os.makedirs(dirname)
+
+    def write_local_file(self, path, file_data):
+        # type: (Text, binary_type) -> None
+        self.mkdirs(path)
+        with open(path, 'wb') as f:   # type: ignore
+            f.write(file_data)
+
+    def upload_files(self, response, resized_image, dst_path_id):
+        # type: (Response, binary_type, Text) -> None
+        dst_file = os.path.join(settings.LOCAL_UPLOADS_DIR, 'avatars', dst_path_id)
+        if resized_image:
+            self.write_local_file(dst_file, resized_image)
+        else:
+            self.write_local_file(dst_file, response.content)
+        self.write_local_file('.'.join((dst_file, 'original')), response.content)
+
+
+class S3Uploader(Uploader):
+    def __init__(self):
+        # type: () -> None
+        super(S3Uploader, self).__init__()
+        conn = S3Connection(settings.S3_KEY, settings.S3_SECRET_KEY)
+        bucket_name = settings.S3_AVATAR_BUCKET
+        self.bucket = conn.get_bucket(bucket_name, validate=False)
+
+    def upload_to_s3(self, path, file_data, headers):
+        # type: (Text, binary_type, Optional[Dict[Text, Text]]) -> None
+        key = Key(self.bucket)
+        key.key = path
+        key.set_contents_from_string(force_str(file_data), headers=headers)
+
+    def upload_files(self, response, resized_image, dst_path_id):
+        # type: (Response, binary_type, Text) -> None
+        headers = None  # type: Optional[Dict[Text, Text]]
+        content_type = response.headers.get(str("Content-Type")) or guess_type(dst_path_id)[0]
+        if content_type:
+            headers = {u'Content-Type': content_type}
+        if resized_image:
+            self.upload_to_s3(dst_path_id, resized_image, headers)
+        else:
+            self.upload_to_s3(dst_path_id, response.content, headers)
+        self.upload_to_s3('.'.join((dst_path_id, 'original')), response.content, headers)
+
+def get_uploader():
+    # type: () -> Uploader
+    if settings.LOCAL_UPLOADS_DIR is None:
+        return S3Uploader()
+    return LocalUploader()
+
+
+def upload_emoji_to_storage(apps, schema_editor):
+    # type: (StateApps, DatabaseSchemaEditor) -> None
+    realm_emoji_model = apps.get_model('zerver', 'RealmEmoji')
+    uploader = get_uploader()  # type: Uploader
+    for emoji in realm_emoji_model.objects.all():
+        file_name = uploader.upload_emoji(emoji.realm_id, emoji.img_url, emoji.name)
+        if file_name is None:
+            logging.warning("ERROR: Could not download emoji %s; please reupload manually" %
+                            (emoji,))
+        emoji.file_name = file_name
+        emoji.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('zerver', '0076_userprofile_emojiset'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='realmemoji',
+            name='file_name',
+            field=models.TextField(db_index=True, null=True),
+        ),
+        migrations.RunPython(upload_emoji_to_storage),
+        migrations.RemoveField(
+            model_name='realmemoji',
+            name='img_url',
+        ),
+    ]

zerver/models.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from typing import Any, DefaultDict, Dict, List, Set, Tuple, TypeVar, Text, \
-    Union, Optional, Sequence, AbstractSet, Pattern, AnyStr, Callable
+    Union, Optional, Sequence, AbstractSet, Pattern, AnyStr, Callable, Iterable
 from typing.re import Match
 from zerver.lib.str_utils import NonBinaryStr
 
@@ -25,7 +25,6 @@ from zerver.lib.cache import cache_with_key, flush_user_profile, flush_realm, \
 from zerver.lib.utils import make_safe_digest, generate_random_token
 from zerver.lib.str_utils import ModelReprMixin
 from django.db import transaction
-from zerver.lib.camo import get_camo_url
 from django.utils.timezone import now as timezone_now
 from django.contrib.sessions.models import Session
 from zerver.lib.timestamp import datetime_to_timestamp
@@ -197,7 +196,7 @@ class Realm(ModelReprMixin, models.Model):
 
     @cache_with_key(get_realm_emoji_cache_key, timeout=3600*24*7)
     def get_emoji(self):
-        # type: () -> Dict[Text, Optional[Dict[str, Text]]]
+        # type: () -> Dict[Text, Optional[Dict[str, Iterable[Text]]]]
         return get_realm_emoji_uncached(self)
 
     def get_admin_users(self):
@@ -394,20 +393,21 @@ class RealmEmoji(ModelReprMixin, models.Model):
     name = models.TextField(validators=[MinLengthValidator(1),
                                         RegexValidator(regex=r'^[0-9a-zA-Z.\-_]+(?<![.\-_])$',
                                                        message=_("Invalid characters in emoji name"))]) # type: Text
-    # URLs start having browser compatibility problem below 2000
+    file_name = models.TextField(db_index=True, null=True) # type: Text
-    # characters, so 1000 seems like a safe limit.
+
-    img_url = models.URLField(max_length=1000) # type: Text
+    PATH_ID_TEMPLATE = "{realm_id}/emoji/{emoji_file_name}"
 
     class Meta(object):
         unique_together = ("realm", "name")
 
     def __unicode__(self):
         # type: () -> Text
-        return u"<RealmEmoji(%s): %s %s>" % (self.realm.string_id, self.name, self.img_url)
+        return u"<RealmEmoji(%s): %s %s>" % (self.realm.string_id, self.name, self.file_name)
 
 def get_realm_emoji_uncached(realm):
-    # type: (Realm) -> Dict[Text, Optional[Dict[str, Text]]]
+    # type: (Realm) -> Dict[Text, Optional[Dict[str, Iterable[Text]]]]
     d = {}
+    from zerver.lib.emoji import get_emoji_url
     for row in RealmEmoji.objects.filter(realm=realm).select_related('author'):
         if row.author:
             author = {
@@ -416,8 +416,7 @@ def get_realm_emoji_uncached(realm):
                 'full_name': row.author.full_name}
         else:
             author = None
-        d[row.name] = dict(source_url=row.img_url,
+        d[row.name] = dict(source_url=get_emoji_url(row.file_name, row.realm_id),
-                           display_url=get_camo_url(row.img_url),
                            author=author)
     return d
 

zerver/tests/test_bugdown.py

@@ -13,6 +13,7 @@ from zerver.lib.actions import (
 )
 from zerver.lib.alert_words import alert_words_in_realm
 from zerver.lib.camo import get_camo_url
+from zerver.lib.emoji import get_emoji_url
 from zerver.lib.message import render_markdown
 from zerver.lib.request import (
     JsonableError,
@@ -478,18 +479,18 @@ class BugdownTest(TestCase):
 
     def test_realm_emoji(self):
         # type: () -> None
-        def emoji_img(name, url):
+        def emoji_img(name, file_name, realm_id):
-            # type: (Text, Text) -> Text
+            # type: (Text, Text, int) -> Text
-            return '<img alt="%s" class="emoji" src="%s" title="%s">' % (name, get_camo_url(url), name)
+            return '<img alt="%s" class="emoji" src="%s" title="%s">' % (
+                name, get_emoji_url(file_name, realm_id), name)
 
         realm = get_realm('zulip')
-        url = "https://zulip.com/test_realm_emoji.png"
+        check_add_realm_emoji(realm, "test", 'test.png')
-        check_add_realm_emoji(realm, "test", url)
 
         # Needs to mock an actual message because that's how bugdown obtains the realm
         msg = Message(sender=get_user_profile_by_email("hamlet@zulip.com"))
         converted = bugdown.convert(":test:", message_realm=realm, message=msg)
-        self.assertEqual(converted, '<p>%s</p>' % (emoji_img(':test:', url)))
+        self.assertEqual(converted, '<p>%s</p>' % (emoji_img(':test:', 'test.png', realm.id)))
 
         do_remove_realm_emoji(realm, 'test')
         converted = bugdown.convert(":test:", message_realm=realm, message=msg)

zerver/tests/test_reactions.py

@@ -5,7 +5,8 @@ import ujson
 from typing import Any, Dict, List
 from six import string_types
 
-from zerver.lib.test_helpers import tornado_redirected_to_list, get_display_recipient
+from zerver.lib.test_helpers import tornado_redirected_to_list, get_display_recipient, \
+    get_test_image_file
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.models import get_realm, get_user_profile_by_email, Recipient, UserMessage
 
@@ -97,9 +98,10 @@ class ReactionEmojiTest(ZulipTestCase):
         """
         sender = 'hamlet@zulip.com'
         emoji_name = 'my_emoji'
-        emoji_data = {'url': 'https://example.com/my_emoji'}
+        with get_test_image_file('img.png') as fp1:
-        result = self.client_put('/json/realm/emoji/my_emoji', info=emoji_data,
+            emoji_data = {'f1': fp1}
-                                 **self.api_auth(sender))
+            result = self.client_put_multipart('/json/realm/emoji/my_emoji', info=emoji_data,
+                                               **self.api_auth(sender))
         self.assert_json_success(result)
         self.assertEqual(200, result.status_code)
 

zerver/tests/test_realm_emoji.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 
 from zerver.lib.actions import get_realm, check_add_realm_emoji
 from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.test_helpers import get_test_image_file
 from zerver.models import RealmEmoji
 import ujson
 
@@ -12,7 +13,7 @@ class RealmEmojiTest(ZulipTestCase):
         # type: () -> None
         self.login("iago@zulip.com")
         realm = get_realm('zulip')
-        check_add_realm_emoji(realm, "my_emoji", "https://example.com/my_emoji")
+        check_add_realm_emoji(realm, "my_emoji", "my_emoji")
         result = self.client_get("/json/realm/emoji")
         self.assert_json_success(result)
         self.assertEqual(200, result.status_code)
@@ -23,7 +24,7 @@ class RealmEmojiTest(ZulipTestCase):
         # type: () -> None
         self.login("iago@zulip.com")
         realm = get_realm('zulip')
-        RealmEmoji.objects.create(realm=realm, name='my_emojy', img_url='https://example.com/my_emoji')
+        RealmEmoji.objects.create(realm=realm, name='my_emojy', file_name='my_emojy')
         result = self.client_get("/json/realm/emoji")
         self.assert_json_success(result)
         content = ujson.loads(result.content)
@@ -36,19 +37,20 @@ class RealmEmojiTest(ZulipTestCase):
         realm = get_realm('zulip')
         realm.add_emoji_by_admins_only = True
         realm.save()
-        check_add_realm_emoji(realm, "my_emoji", "https://example.com/my_emoji")
+        check_add_realm_emoji(realm, 'my_emojy', 'my_emojy')
         result = self.client_get("/json/realm/emoji")
         self.assert_json_success(result)
         content = ujson.loads(result.content)
         self.assertEqual(len(content["emoji"]), 1)
-        self.assertIsNone(content["emoji"]['my_emoji']['author'])
+        self.assertIsNone(content["emoji"]['my_emojy']['author'])
 
     def test_upload(self):
         # type: () -> None
         email = "iago@zulip.com"
         self.login(email)
-        data = {"url": "https://example.com/my_emoji"}
+        with get_test_image_file('img.png') as fp1:
-        result = self.client_put("/json/realm/emoji/my_emoji", data)
+            emoji_data = {'f1': fp1}
+            result = self.client_put_multipart('/json/realm/emoji/my_emoji', info=emoji_data)
         self.assert_json_success(result)
         self.assertEqual(200, result.status_code)
         emoji = RealmEmoji.objects.get(name="my_emoji")
@@ -65,14 +67,15 @@ class RealmEmojiTest(ZulipTestCase):
         realm_emoji = RealmEmoji.objects.get(realm=get_realm('zulip'))
         self.assertEqual(
             str(realm_emoji),
-            '<RealmEmoji(zulip): my_emoji https://example.com/my_emoji>'
+            '<RealmEmoji(zulip): my_emoji my_emoji.png>'
         )
 
     def test_upload_exception(self):
         # type: () -> None
         self.login("iago@zulip.com")
-        data = {"url": "https://example.com/my_emoji"}
+        with get_test_image_file('img.png') as fp1:
-        result = self.client_put("/json/realm/emoji/my_em*oji", info=data)
+            emoji_data = {'f1': fp1}
+            result = self.client_put_multipart('/json/realm/emoji/my_em*oji', info=emoji_data)
         self.assert_json_error(result, 'Invalid characters in emoji name')
 
     def test_upload_admins_only(self):
@@ -81,15 +84,16 @@ class RealmEmojiTest(ZulipTestCase):
         realm = get_realm('zulip')
         realm.add_emoji_by_admins_only = True
         realm.save()
-        data = {"url": "https://example.com/my_emoji"}
+        with get_test_image_file('img.png') as fp1:
-        result = self.client_put("/json/realm/emoji/my_emoji", info=data)
+            emoji_data = {'f1': fp1}
+            result = self.client_put_multipart('/json/realm/emoji/my_emoji', info=emoji_data)
         self.assert_json_error(result, 'Must be a realm administrator')
 
     def test_delete(self):
         # type: () -> None
         self.login("iago@zulip.com")
         realm = get_realm('zulip')
-        check_add_realm_emoji(realm, "my_emoji", "https://example.com/my_emoji")
+        check_add_realm_emoji(realm, "my_emoji", "my_emoji.png")
         result = self.client_delete("/json/realm/emoji/my_emoji")
         self.assert_json_success(result)
 
@@ -104,7 +108,7 @@ class RealmEmojiTest(ZulipTestCase):
         realm = get_realm('zulip')
         realm.add_emoji_by_admins_only = True
         realm.save()
-        check_add_realm_emoji(realm, "my_emoji", "https://example.com/my_emoji")
+        check_add_realm_emoji(realm, "my_emoji", "my_emoji.png")
         result = self.client_delete("/json/realm/emoji/my_emoji")
         self.assert_json_error(result, 'Must be a realm administrator')
 
@@ -113,3 +117,29 @@ class RealmEmojiTest(ZulipTestCase):
         self.login("iago@zulip.com")
         result = self.client_delete("/json/realm/emoji/invalid_emoji")
         self.assert_json_error(result, "Emoji 'invalid_emoji' does not exist")
+
+    def test_multiple_upload(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        with get_test_image_file('img.png') as fp1, get_test_image_file('img.png') as fp2:
+            result = self.client_put_multipart('/json/realm/emoji/my_emoji', {'f1': fp1, 'f2': fp2})
+        self.assert_json_error(result, 'You must upload exactly one file.')
+
+    def test_emoji_upload_file_size_error(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        with get_test_image_file('img.png') as fp:
+            with self.settings(MAX_EMOJI_FILE_SIZE=0):
+                result = self.client_put_multipart('/json/realm/emoji/my_emoji', {'file': fp})
+        self.assert_json_error(result, 'Uploaded file is larger than the allowed limit of 0 MB')
+
+    def test_upload_already_existed_emoji(self):
+        # type: () -> None
+        self.login("iago@zulip.com")
+        with get_test_image_file('img.png') as fp1:
+            emoji_data = {'f1': fp1}
+            self.client_put_multipart('/json/realm/emoji/my_emoji', info=emoji_data)
+        with get_test_image_file('img.png') as fp1:
+                emoji_data = {'f1': fp1}
+                result = self.client_put_multipart('/json/realm/emoji/my_emoji', info=emoji_data)
+        self.assert_json_error(result, 'Realm emoji with this Realm and Name already exists.')

zerver/views/realm_emoji.py

@@ -1,15 +1,20 @@
 from __future__ import absolute_import
 
+from django.conf import settings
+from django.core.exceptions import ValidationError
 from django.http import HttpRequest, HttpResponse
 from django.utils.translation import ugettext as _
 from typing import Text
 
+from zerver.lib.upload import upload_emoji_image
 from zerver.models import UserProfile
-from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name, check_valid_emoji
+from zerver.lib.emoji import check_emoji_admin, check_valid_emoji_name, check_valid_emoji, \
+    get_emoji_file_name
 from zerver.lib.request import JsonableError, REQ, has_request_variables
-from zerver.lib.response import json_success
+from zerver.lib.response import json_success, json_error
 from zerver.lib.actions import check_add_realm_emoji, do_remove_realm_emoji
 
+
 def list_emoji(request, user_profile):
     # type: (HttpRequest, UserProfile) -> HttpResponse
 
@@ -17,14 +22,27 @@ def list_emoji(request, user_profile):
     # emoji is public.
     return json_success({'emoji': user_profile.realm.get_emoji()})
 
+
 @has_request_variables
-def upload_emoji(request, user_profile, emoji_name, url=REQ()):
+def upload_emoji(request, user_profile, emoji_name=REQ()):
-    # type: (HttpRequest, UserProfile, Text, Text) -> HttpResponse
+    # type: (HttpRequest, UserProfile, Text) -> HttpResponse
     check_valid_emoji_name(emoji_name)
     check_emoji_admin(user_profile)
-    check_add_realm_emoji(user_profile.realm, emoji_name, url, author=user_profile)
+    if len(request.FILES) != 1:
+        return json_error(_("You must upload exactly one file."))
+    emoji_file = list(request.FILES.values())[0]
+    if (settings.MAX_EMOJI_FILE_SIZE * 1024 * 1024) < emoji_file.size:
+        return json_error(_("Uploaded file is larger than the allowed limit of %s MB") % (
+            settings.MAX_EMOJI_FILE_SIZE))
+    emoji_file_name = get_emoji_file_name(emoji_file.name, emoji_name)
+    upload_emoji_image(emoji_file, emoji_file_name, user_profile)
+    try:
+        check_add_realm_emoji(user_profile.realm, emoji_name, emoji_file_name, author=user_profile)
+    except ValidationError as e:
+        return json_error(e.messages[0])
     return json_success()
 
+
 def delete_emoji(request, user_profile, emoji_name):
     # type: (HttpRequest, UserProfile, Text) -> HttpResponse
     check_emoji_admin(user_profile)

zproject/settings.py

@@ -123,6 +123,7 @@ DEFAULT_SETTINGS = {'TWITTER_CONSUMER_KEY': '',
                     'MAX_FILE_UPLOAD_SIZE': 25,
                     'MAX_AVATAR_FILE_SIZE': 5,
                     'MAX_ICON_FILE_SIZE': 5,
+                    'MAX_EMOJI_FILE_SIZE': 5,
                     'ERROR_REPORTING': True,
                     'BROWSER_ERROR_REPORTING': False,
                     'STAGING_ERROR_NOTIFICATIONS': False,