emoji: Add support for translating emoticons.

Add `translate_emoticons` to `prop_types` and `expected_keys`.
Furthermore, create a emoji-translating Markdown inline pattern.

Also use a JavaScript version of `translate_emoticons` and then use
this function during Markdown previews and as a preprocessor. This
is only needed for previews, because usually emoticon translation
happens on the backend after sending.

Add tests for emoticon translation, a settings UI, and a /help/ page
as well.

Tweaked by tabbott to fix various test failurse as well as how this
handles whitespace, requiring emoticons to not have adjacent
characters.

Fixes #1768.

frontend_tests/node_tests/dispatch.js

@@ -413,6 +413,12 @@ var event_fixtures = {
         setting: true,
     },
 
+    update_display_settings__translate_emoticons: {
+        type: 'update_display_settings',
+        setting_name: 'translate_emoticons',
+        setting: true,
+    },
+
     update_global_notifications: {
         type: 'update_global_notifications',
         notification_name: 'enable_stream_sounds',
@@ -813,6 +819,10 @@ with_overrides(function (override) {
     dispatch(event);
     assert_same(page_params.twenty_four_hour_time, true);
 
+    event = event_fixtures.update_display_settings__translate_emoticons;
+    page_params.translate_emoticons = false;
+    dispatch(event);
+    assert_same(page_params.translate_emoticons, true);
 });
 
 with_overrides(function (override) {

frontend_tests/node_tests/emoji.js

@@ -6,6 +6,8 @@ set_global('upload_widget', {});
 
 zrequire('emoji_codes', 'generated/emoji/emoji_codes');
 zrequire('emoji');
+zrequire('markdown');
+zrequire('util');
 
 (function test_build_emoji_upload_widget() {
     var build_widget_stub = false;
@@ -74,3 +76,17 @@ zrequire('emoji');
     emoji.get_canonical_name('non_existent');
     assert(errored);
 }());
+
+(function test_translate_emoticons_to_names() {
+  global.emoji_codes = {
+      codepoint_to_name: {
+          '1f603': 'smiley',
+      },
+  };
+
+  var test_text = 'Testing :)';
+  var expected = 'Testing :smiley:';
+  var result = emoji.translate_emoticons_to_names(test_text);
+
+  assert.equal(expected, result);
+}());

frontend_tests/node_tests/markdown.js

@@ -41,6 +41,7 @@ set_global('page_params', {
             "https://zone_%(zone)s.zulip.net/ticket/%(id)s",
         ],
     ],
+    translate_emoticons: false,
 });
 
 set_global('blueslip', {error: function () {}});
@@ -304,6 +305,17 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver
 
         assert.equal(expected, output);
     });
+
+    // Here to arrange 100% test coverage for the new emoticons code
+    // path.  TODO: Have a better way to test this setting in both
+    // states, once we implement the local echo feature properly.
+    // Probably a good technique would be to support toggling the
+    // page_params setting inside the `test_cases.forEach` loop above.
+    page_params.translate_emoticons = true;
+    var message = {raw_content: ":)"};
+    markdown.apply_markdown(message);
+    assert.equal('<p><span class="emoji emoji-1f603" title="smiley">:smiley:</span></p>', message.content);
+    page_params.translate_emoticons = false;
 }());
 
 (function test_subject_links() {

static/js/compose.js

@@ -885,6 +885,7 @@ exports.initialize = function () {
         } else {
             preview_html = rendered_content;
         }
+
         $("#preview_content").html(preview_html);
         if (page_params.emojiset === "text") {
             $("#preview_content").find(".emoji").replaceWith(function () {

static/js/emoji.js

@@ -17,6 +17,18 @@ var zulip_emoji = {
     deactivated: false,
 };
 
+// Emoticons, and which emoji they should become (without colons). Duplicate
+// emoji are allowed. Changes here should be mimicked in `zerver/lib/emoji.py`
+// and `templates/zerver/help/enable-emoticon-translations.md`.
+var EMOTICON_CONVERSIONS = {
+    ':)': 'smiley',
+    '(:': 'smiley',
+    ':(': 'slightly_frowning_face',
+    '<3': 'heart',
+    ':|': 'expressionless',
+    ':/': 'confused',
+};
+
 exports.update_emojis = function update_emojis(realm_emojis) {
     // exports.all_realm_emojis is emptied before adding the realm-specific emoji to it.
     // This makes sure that in case of deletion, the deleted realm_emojis don't
@@ -105,6 +117,22 @@ exports.get_canonical_name = function (emoji_name) {
     return emoji_codes.codepoint_to_name[codepoint];
 };
 
+// Translates emoticons in a string to their colon syntax.
+exports.translate_emoticons_to_names = function translate_emoticons_to_names(text) {
+    var translated = text;
+
+    for (var emoticon in EMOTICON_CONVERSIONS) {
+        if (EMOTICON_CONVERSIONS.hasOwnProperty(emoticon)) {
+            var emoticon_reg_ex = new RegExp(util.escape_regexp(emoticon), "g");
+            translated = translated.replace(
+                emoticon_reg_ex,
+                ':' + EMOTICON_CONVERSIONS[emoticon] + ':');
+        }
+    }
+
+    return translated;
+};
+
 return exports;
 }());
 if (typeof module !== 'undefined') {

static/js/markdown.js

@@ -358,6 +358,16 @@ exports.initialize = function () {
         return fenced_code.process_fenced_code(src);
     }
 
+    function preprocess_translate_emoticons(src) {
+        if (!page_params.translate_emoticons) {
+            return src;
+        }
+
+        // In this scenario, the message has to be from the user, so the only
+        // requirement should be that they have the setting on.
+        return emoji.translate_emoticons_to_names(src);
+    }
+
     // Disable ordered lists
     // We used GFM + tables, so replace the list start regex for that ruleset
     // We remove the |[\d+]\. that matches the numbering in a numbered list
@@ -406,7 +416,11 @@ exports.initialize = function () {
         realmFilterHandler: handleRealmFilter,
         texHandler: handleTex,
         renderer: r,
-        preprocessors: [preprocess_code_blocks, preprocess_auto_olists],
+        preprocessors: [
+            preprocess_code_blocks,
+            preprocess_auto_olists,
+            preprocess_translate_emoticons,
+        ],
     });
 
 };

static/js/server_events_dispatch.js

@@ -288,6 +288,7 @@ exports.dispatch_normal_event = function dispatch_normal_event(event) {
             'left_side_userlist',
             'timezone',
             'twenty_four_hour_time',
+            'translate_emoticons',
         ];
         if (_.contains(user_display_settings, event.setting_name)) {
             page_params[event.setting_name] = event.setting;

static/js/settings_display.js

@@ -184,6 +184,31 @@ exports.set_up = function () {
             },
         });
     });
+
+    $("#translate_emoticons").change(function () {
+        var data = {};
+        var setting_value = $("#translate_emoticons").is(":checked");
+        data.translate_emoticons = JSON.stringify(setting_value);
+        var context = {};
+        if (data.translate_emoticons === "true") {
+            context.new_mode = i18n.t("be");
+        } else {
+            context.new_mode = i18n.t("not be");
+        }
+
+        channel.patch({
+            url: '/json/settings/display',
+            data: data,
+            success: function () {
+                ui_report.success(i18n.t("Emoticons will now __new_mode__ translated!", context),
+                                  $('#display-settings-status').expectOne());
+            },
+            error: function (xhr) {
+                ui_report.error(i18n.t("Error updating emoticon translation setting"), xhr,
+                                $('#display-settings-status').expectOne());
+            },
+        });
+    });
 };
 
 exports.report_emojiset_change = function () {
@@ -214,6 +239,7 @@ function _update_page() {
     $("#twenty_four_hour_time").prop('checked', page_params.twenty_four_hour_time);
     $("#left_side_userlist").prop('checked', page_params.left_side_userlist);
     $("#default_language_name").text(page_params.default_language_name);
+    $("#translate_emoticons").prop('checked', page_params.translate_emoticons);
 }
 
 exports.update_page = function () {

static/templates/settings/display-settings.handlebars

@@ -78,7 +78,21 @@
             </select>
         </div>
 
-        <h3>Emoji style</h3>
+        <h3 class="light">Emoji style</h3>
+
+        <div class="input-group side-padded-container">
+            <label class="checkbox">
+                <input type="checkbox" name="translate_emoticons" id="translate_emoticons"
+                       {{#if page_params.translate_emoticons}}
+                       checked="checked"
+                       {{/if}} />
+                <span></span>
+            </label>
+            <label for="translate_emoticons" class="inline-block">
+                {{t "Translate emoticons (convert <code>:)</code> to 😃 in messages)" }}
+            </label>
+        </div>
+
         <div class="input-group side-padded-container">
             <div class="emojiset_choices grey-box">
                 {{#each page_params.emojiset_choices }}

templates/zerver/help/enable-emoticon-translations.md

@@ -0,0 +1,92 @@
+# Enable emoticon translation
+
+If you use emoticons like `:)` or `:/`, you can have them translated into
+emoji equivalents like
+<img
+    src="/static/generated/emoji/images/emoji/smile.png"
+    alt="smiley"
+    style="width: 3%;"
+/>
+or
+<img
+    src="/static/generated/emoji/images/emoji/slightly_frowning_face.png"
+    alt="slightly_frowning_face"
+    style="width: 3%;"
+/>
+automatically by Zulip.
+
+{!go-to-the.md!} [Display settings](/#settings/display-settings)
+{!settings.md!}
+
+2. Select the option labeled
+   **Translate emoticons (convert `:)` to 😃 in messages)**.
+
+Then whenever you send a message with a supported emoticon, it will be
+translated into an emoji.
+
+## Current emoticon conversions
+
+<table>
+    <thead>
+        <tr>
+            <th align="center">Emoticon</th>
+            <th align="center">Emoji</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td align="center"><code>:)</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/smiley.png"
+                    alt="smiley"
+                    style="width: 30%;">
+            </td>
+        </tr>
+        <tr>
+            <td align="center"><code>(:</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/smiley.png"
+                    alt="smiley"
+                    style="width: 30%;">
+            </td>
+        </tr>
+        <tr>
+            <td align="center"><code>:(</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/slightly_frowning_face.png"
+                    alt="slightly_frowning_face"
+                    style="width: 30%;">
+            </td>
+        </tr>
+        <tr>
+            <td align="center"><code>&lt;3</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/heart.png"
+                    alt="heart"
+                    style="width: 30%;">
+            </td>
+        </tr>
+        <tr>
+            <td align="center"><code>:|</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/expressionless.png"
+                    alt="expressionless"
+                    style="width: 30%;">
+            </td>
+        </tr>
+        <tr>
+            <td align="center"><code>:/</code></td>
+            <td align="center">
+                <img
+                    src="/static/generated/emoji/images/emoji/confused.png"
+                    alt="confused"
+                    style="width: 30%;">
+            </td>
+        </tr>
+    </tbody>
+</table>

templates/zerver/help/include/sidebar.md

@@ -99,6 +99,7 @@
 * [Bots and integrations](/help/add-a-bot-or-integration)
 * [Enable high contrast mode](/help/enable-high-contrast-mode)
 * [Enable night mode](/help/enable-night-mode)
+* [Enable emoticon translations](/help/enable-emoticon-translations)
 * [Display the buddy list on narrow screens](/help/move-the-users-list-to-the-left-sidebar)
 * [View organization statistics](/help/analytics)
 

zerver/fixtures/markdown_test_cases.json

@@ -336,9 +336,28 @@
     },
     {
       "name": "many_emoji",
-      "input":  "test :smile: again :poop:\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
+      "input":  "test :smile: again :poop:\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
-      "expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
+      "expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
-      "text_content": "test \ud83d\ude04 again \ud83d\udca9\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
+      "text_content": "test \ud83d\ude04 again \ud83d\udca9\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
+    },
+    {
+      "name": "translate_emoticons",
+      "input":  ":) foo :( bar <3 with space : ) real emoji :smiley:",
+      "expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slightly frowning face\">:slightly_frowning_face:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
+      "marked_expected_output": "<p>:) foo :( bar &lt;3 with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
+      "text_content": "\ud83d\ude03 foo \ud83d\ude41 bar \u2764 with space : ) real emoji \ud83d\ude03"
+    },
+    {
+      "name": "translate_emoticons_whitepsace",
+      "input":  "a:) ;)b",
+      "expected_output": "<p>a:) ;)b</p>",
+      "text_content": "a:) ;)b"
+    },
+    {
+      "name": "translate_emoticons_in_code",
+      "input":  "`:)`",
+      "expected_output": "<p><code>:)</code></p>",
+      "text_content": ":)"
     },
     {
       "name": "random_emoji_1",

zerver/lib/bugdown/__init__.py

@@ -32,6 +32,7 @@ from markdown.extensions import codehilite
 from zerver.lib.bugdown import fenced_code
 from zerver.lib.bugdown.fenced_code import FENCE_RE
 from zerver.lib.camo import get_camo_url
+from zerver.lib.emoji import translate_emoticons, emoticon_regex
 from zerver.lib.mention import possible_mentions, \
     possible_user_group_mentions, extract_user_group
 from zerver.lib.notifications import encode_stream
@@ -998,6 +999,19 @@ def unicode_emoji_to_codepoint(unicode_emoji: Text) -> Text:
         codepoint = '0' + codepoint
     return codepoint
 
+class EmoticonTranslation(markdown.inlinepatterns.Pattern):
+    """ Translates emoticons like `:)` into emoji like `:smile:`. """
+    def handleMatch(self, match: Match[Text]) -> Optional[Element]:
+        # If there is `db_data` and it is false, then don't do translating.
+        # If there is no `db_data`, such as during tests, translate.
+        if db_data is not None and not db_data['translate_emoticons']:
+            return None
+
+        emoticon = match.group('emoticon')
+        translated = translate_emoticons(emoticon)
+        name = translated[1:-1]
+        return make_emoji(name_to_codepoint[name], translated)
+
 class UnicodeEmoji(markdown.inlinepatterns.Pattern):
     def handleMatch(self, match: Match[Text]) -> Optional[Element]:
         orig_syntax = match.group('syntax')
@@ -1578,6 +1592,7 @@ class Bugdown(markdown.Extension):
             Tex(r'\B(?<!\$)\$\$(?P<body>[^\n_$](\\\$|[^$\n])*)\$\$(?!\$)\B'),
             '>backtick')
         md.inlinePatterns.add('emoji', Emoji(EMOJI_REGEX), '_end')
+        md.inlinePatterns.add('translate_emoticons', EmoticonTranslation(emoticon_regex), '>emoji')
         md.inlinePatterns.add('unicodeemoji', UnicodeEmoji(unicode_emoji_regex), '_end')
         md.inlinePatterns.add('link', AtomicLinkPattern(markdown.inlinepatterns.LINK_RE, md), '>avatar')
 
@@ -1956,6 +1971,7 @@ def do_convert(content: Text,
             'realm_uri': message_realm.uri,
             'sent_by_bot': sent_by_bot,
             'stream_names': stream_name_info,
+            'translate_emoticons': message.sender.translate_emoticons,
         }
 
     try:

zerver/lib/emoji.py

@@ -14,6 +14,31 @@ from zerver.models import Reaction, Realm, RealmEmoji, UserProfile
 NAME_TO_CODEPOINT_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "name_to_codepoint.json")
 CODEPOINT_TO_NAME_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "codepoint_to_name.json")
 
+# Emoticons and which emoji they should become. Duplicate emoji are allowed.
+# Changes here should be mimicked in `static/js/emoji.js`
+# and `templates/zerver/help/enable-emoticon-translations.md`.
+EMOTICON_CONVERSIONS = {
+    ':)': ':smiley:',
+    '(:': ':smiley:',
+    ':(': ':slightly_frowning_face:',
+    '<3': ':heart:',
+    ':|': ':expressionless:',
+    ':/': ':confused:',
+}
+
+possible_emoticons = EMOTICON_CONVERSIONS.keys()
+possible_emoticon_regexes = map(re.escape, possible_emoticons)  # type: ignore # AnyStr/str issues
+emoticon_regex = '(?<![^\s])(?P<emoticon>(' + ')|('.join(possible_emoticon_regexes) + '))(?![\S])'  # type: ignore # annoying
+
+# Translates emoticons to their colon syntax, e.g. `:smiley:`.
+def translate_emoticons(text: Text) -> Text:
+    translated = text
+
+    for emoticon in EMOTICON_CONVERSIONS:
+        translated = re.sub(re.escape(emoticon), EMOTICON_CONVERSIONS[emoticon], translated)
+
+    return translated
+
 with open(NAME_TO_CODEPOINT_PATH) as fp:
     name_to_codepoint = ujson.load(fp)
 

zerver/migrations/0142_userprofile_translate_emoticons.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2018-02-19 22:27
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zerver', '0141_change_usergroup_description_to_textfield'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userprofile',
+            name='translate_emoticons',
+            field=models.BooleanField(default=False),
+        ),
+    ]

zerver/models.py

@@ -588,6 +588,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin):
     default_language = models.CharField(default=u'en', max_length=MAX_LANGUAGE_ID_LENGTH)  # type: Text
     high_contrast_mode = models.BooleanField(default=False)  # type: bool
     night_mode = models.BooleanField(default=False)  # type: bool
+    translate_emoticons = models.BooleanField(default=False)  # type: bool
 
     # Hours to wait before sending another email to a user
     EMAIL_REMINDER_WAITPERIOD = 24
@@ -650,6 +651,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin):
         twenty_four_hour_time=bool,
         high_contrast_mode=bool,
         night_mode=bool,
+        translate_emoticons=bool,
     )
 
     notification_setting_types = dict(

zerver/tests/test_bugdown.py

@@ -4,6 +4,7 @@ from django.test import TestCase, override_settings
 
 from zerver.lib import bugdown
 from zerver.lib.actions import (
+    do_set_user_display_setting,
     do_remove_realm_emoji,
     do_set_alert_words,
     get_realm,
@@ -617,6 +618,16 @@ class BugdownTest(ZulipTestCase):
         converted = bugdown_convert(msg)
         self.assertEqual(converted, u'<p><span class="emoji emoji-2615" title="coffee">:coffee:</span><span class="emoji emoji-2615" title="coffee">:coffee:</span></p>')
 
+    def test_no_translate_emoticons_if_off(self) -> None:
+        user_profile = self.example_user('othello')
+        do_set_user_display_setting(user_profile, 'translate_emoticons', False)
+        msg = Message(sender=user_profile, sending_client=get_client("test"))
+
+        content = u':)'
+        expected = u'<p>:)</p>'
+        converted = render_markdown(msg, content)
+        self.assertEqual(converted, expected)
+
     def test_same_markup(self) -> None:
         msg = u'\u2615'  # ☕
         unicode_converted = bugdown_convert(msg)

zerver/tests/test_home.py

@@ -156,6 +156,7 @@ class HomeTest(ZulipTestCase):
             "subscriptions",
             "test_suite",
             "timezone",
+            "translate_emoticons",
             "twenty_four_hour_time",
             "unread_msgs",
             "unsubscribed",

zerver/views/user_settings.py

@@ -129,6 +129,7 @@ def update_display_settings_backend(
         twenty_four_hour_time: Optional[bool]=REQ(validator=check_bool, default=None),
         high_contrast_mode: Optional[bool]=REQ(validator=check_bool, default=None),
         night_mode: Optional[bool]=REQ(validator=check_bool, default=None),
+        translate_emoticons: Optional[bool]=REQ(validator=check_bool, default=None),
         default_language: Optional[bool]=REQ(validator=check_string, default=None),
         left_side_userlist: Optional[bool]=REQ(validator=check_bool, default=None),
         emojiset: Optional[str]=REQ(validator=check_string, default=None),