diff --git a/frontend_tests/node_tests/dispatch.js b/frontend_tests/node_tests/dispatch.js index 025d0ccad5..d952d48afa 100644 --- a/frontend_tests/node_tests/dispatch.js +++ b/frontend_tests/node_tests/dispatch.js @@ -413,6 +413,12 @@ var event_fixtures = { setting: true, }, + update_display_settings__translate_emoticons: { + type: 'update_display_settings', + setting_name: 'translate_emoticons', + setting: true, + }, + update_global_notifications: { type: 'update_global_notifications', notification_name: 'enable_stream_sounds', @@ -813,6 +819,10 @@ with_overrides(function (override) { dispatch(event); assert_same(page_params.twenty_four_hour_time, true); + event = event_fixtures.update_display_settings__translate_emoticons; + page_params.translate_emoticons = false; + dispatch(event); + assert_same(page_params.translate_emoticons, true); }); with_overrides(function (override) { diff --git a/frontend_tests/node_tests/emoji.js b/frontend_tests/node_tests/emoji.js index bf93338ce0..d9cf56112b 100644 --- a/frontend_tests/node_tests/emoji.js +++ b/frontend_tests/node_tests/emoji.js @@ -6,6 +6,8 @@ set_global('upload_widget', {}); zrequire('emoji_codes', 'generated/emoji/emoji_codes'); zrequire('emoji'); +zrequire('markdown'); +zrequire('util'); (function test_build_emoji_upload_widget() { var build_widget_stub = false; @@ -74,3 +76,17 @@ zrequire('emoji'); emoji.get_canonical_name('non_existent'); assert(errored); }()); + +(function test_translate_emoticons_to_names() { + global.emoji_codes = { + codepoint_to_name: { + '1f603': 'smiley', + }, + }; + + var test_text = 'Testing :)'; + var expected = 'Testing :smiley:'; + var result = emoji.translate_emoticons_to_names(test_text); + + assert.equal(expected, result); +}()); diff --git a/frontend_tests/node_tests/markdown.js b/frontend_tests/node_tests/markdown.js index 6f6fa49bc3..e2d8dcbe8a 100644 --- a/frontend_tests/node_tests/markdown.js +++ b/frontend_tests/node_tests/markdown.js @@ -41,6 +41,7 @@ set_global('page_params', { "https://zone_%(zone)s.zulip.net/ticket/%(id)s", ], ], + translate_emoticons: false, }); set_global('blueslip', {error: function () {}}); @@ -304,6 +305,17 @@ var bugdown_data = JSON.parse(fs.readFileSync(path.join(__dirname, '../../zerver assert.equal(expected, output); }); + + // Here to arrange 100% test coverage for the new emoticons code + // path. TODO: Have a better way to test this setting in both + // states, once we implement the local echo feature properly. + // Probably a good technique would be to support toggling the + // page_params setting inside the `test_cases.forEach` loop above. + page_params.translate_emoticons = true; + var message = {raw_content: ":)"}; + markdown.apply_markdown(message); + assert.equal('

:smiley:

', message.content); + page_params.translate_emoticons = false; }()); (function test_subject_links() { diff --git a/static/js/compose.js b/static/js/compose.js index ed99fd3c81..6013ff770d 100644 --- a/static/js/compose.js +++ b/static/js/compose.js @@ -885,6 +885,7 @@ exports.initialize = function () { } else { preview_html = rendered_content; } + $("#preview_content").html(preview_html); if (page_params.emojiset === "text") { $("#preview_content").find(".emoji").replaceWith(function () { diff --git a/static/js/emoji.js b/static/js/emoji.js index 1d2cf2e505..0f0ab7edc2 100644 --- a/static/js/emoji.js +++ b/static/js/emoji.js @@ -17,6 +17,18 @@ var zulip_emoji = { deactivated: false, }; +// Emoticons, and which emoji they should become (without colons). Duplicate +// emoji are allowed. Changes here should be mimicked in `zerver/lib/emoji.py` +// and `templates/zerver/help/enable-emoticon-translations.md`. +var EMOTICON_CONVERSIONS = { + ':)': 'smiley', + '(:': 'smiley', + ':(': 'slightly_frowning_face', + '<3': 'heart', + ':|': 'expressionless', + ':/': 'confused', +}; + exports.update_emojis = function update_emojis(realm_emojis) { // exports.all_realm_emojis is emptied before adding the realm-specific emoji to it. // This makes sure that in case of deletion, the deleted realm_emojis don't @@ -105,6 +117,22 @@ exports.get_canonical_name = function (emoji_name) { return emoji_codes.codepoint_to_name[codepoint]; }; +// Translates emoticons in a string to their colon syntax. +exports.translate_emoticons_to_names = function translate_emoticons_to_names(text) { + var translated = text; + + for (var emoticon in EMOTICON_CONVERSIONS) { + if (EMOTICON_CONVERSIONS.hasOwnProperty(emoticon)) { + var emoticon_reg_ex = new RegExp(util.escape_regexp(emoticon), "g"); + translated = translated.replace( + emoticon_reg_ex, + ':' + EMOTICON_CONVERSIONS[emoticon] + ':'); + } + } + + return translated; +}; + return exports; }()); if (typeof module !== 'undefined') { diff --git a/static/js/markdown.js b/static/js/markdown.js index 76ac52e81d..c436e73ba6 100644 --- a/static/js/markdown.js +++ b/static/js/markdown.js @@ -358,6 +358,16 @@ exports.initialize = function () { return fenced_code.process_fenced_code(src); } + function preprocess_translate_emoticons(src) { + if (!page_params.translate_emoticons) { + return src; + } + + // In this scenario, the message has to be from the user, so the only + // requirement should be that they have the setting on. + return emoji.translate_emoticons_to_names(src); + } + // Disable ordered lists // We used GFM + tables, so replace the list start regex for that ruleset // We remove the |[\d+]\. that matches the numbering in a numbered list @@ -406,7 +416,11 @@ exports.initialize = function () { realmFilterHandler: handleRealmFilter, texHandler: handleTex, renderer: r, - preprocessors: [preprocess_code_blocks, preprocess_auto_olists], + preprocessors: [ + preprocess_code_blocks, + preprocess_auto_olists, + preprocess_translate_emoticons, + ], }); }; diff --git a/static/js/server_events_dispatch.js b/static/js/server_events_dispatch.js index f352727254..e06e55d823 100644 --- a/static/js/server_events_dispatch.js +++ b/static/js/server_events_dispatch.js @@ -288,6 +288,7 @@ exports.dispatch_normal_event = function dispatch_normal_event(event) { 'left_side_userlist', 'timezone', 'twenty_four_hour_time', + 'translate_emoticons', ]; if (_.contains(user_display_settings, event.setting_name)) { page_params[event.setting_name] = event.setting; diff --git a/static/js/settings_display.js b/static/js/settings_display.js index 99960d0a51..ed5d16f1da 100644 --- a/static/js/settings_display.js +++ b/static/js/settings_display.js @@ -184,6 +184,31 @@ exports.set_up = function () { }, }); }); + + $("#translate_emoticons").change(function () { + var data = {}; + var setting_value = $("#translate_emoticons").is(":checked"); + data.translate_emoticons = JSON.stringify(setting_value); + var context = {}; + if (data.translate_emoticons === "true") { + context.new_mode = i18n.t("be"); + } else { + context.new_mode = i18n.t("not be"); + } + + channel.patch({ + url: '/json/settings/display', + data: data, + success: function () { + ui_report.success(i18n.t("Emoticons will now __new_mode__ translated!", context), + $('#display-settings-status').expectOne()); + }, + error: function (xhr) { + ui_report.error(i18n.t("Error updating emoticon translation setting"), xhr, + $('#display-settings-status').expectOne()); + }, + }); + }); }; exports.report_emojiset_change = function () { @@ -214,6 +239,7 @@ function _update_page() { $("#twenty_four_hour_time").prop('checked', page_params.twenty_four_hour_time); $("#left_side_userlist").prop('checked', page_params.left_side_userlist); $("#default_language_name").text(page_params.default_language_name); + $("#translate_emoticons").prop('checked', page_params.translate_emoticons); } exports.update_page = function () { diff --git a/static/templates/settings/display-settings.handlebars b/static/templates/settings/display-settings.handlebars index 18fd210f73..389f5f26cf 100644 --- a/static/templates/settings/display-settings.handlebars +++ b/static/templates/settings/display-settings.handlebars @@ -78,7 +78,21 @@ -

Emoji style

+

Emoji style

+ +
+ + +
+
{{#each page_params.emojiset_choices }} diff --git a/templates/zerver/help/enable-emoticon-translations.md b/templates/zerver/help/enable-emoticon-translations.md new file mode 100644 index 0000000000..8ddea00eec --- /dev/null +++ b/templates/zerver/help/enable-emoticon-translations.md @@ -0,0 +1,92 @@ +# Enable emoticon translation + +If you use emoticons like `:)` or `:/`, you can have them translated into +emoji equivalents like +smiley +or +slightly_frowning_face +automatically by Zulip. + +{!go-to-the.md!} [Display settings](/#settings/display-settings) +{!settings.md!} + +2. Select the option labeled + **Translate emoticons (convert `:)` to 😃 in messages)**. + +Then whenever you send a message with a supported emoticon, it will be +translated into an emoji. + +## Current emoticon conversions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EmoticonEmoji
:) + smiley +
(: + smiley +
:( + slightly_frowning_face +
<3 + heart +
:| + expressionless +
:/ + confused +
diff --git a/templates/zerver/help/include/sidebar.md b/templates/zerver/help/include/sidebar.md index 81c5d4a6ef..c6777813e8 100644 --- a/templates/zerver/help/include/sidebar.md +++ b/templates/zerver/help/include/sidebar.md @@ -99,6 +99,7 @@ * [Bots and integrations](/help/add-a-bot-or-integration) * [Enable high contrast mode](/help/enable-high-contrast-mode) * [Enable night mode](/help/enable-night-mode) +* [Enable emoticon translations](/help/enable-emoticon-translations) * [Display the buddy list on narrow screens](/help/move-the-users-list-to-the-left-sidebar) * [View organization statistics](/help/analytics) diff --git a/zerver/fixtures/markdown_test_cases.json b/zerver/fixtures/markdown_test_cases.json index c70d0a219f..8b2085149d 100644 --- a/zerver/fixtures/markdown_test_cases.json +++ b/zerver/fixtures/markdown_test_cases.json @@ -336,9 +336,28 @@ }, { "name": "many_emoji", - "input": "test :smile: again :poop:\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:", - "expected_output": "

test :smile: again :poop:
\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:

", - "text_content": "test \ud83d\ude04 again \ud83d\udca9\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:" + "input": "test :smile: again :poop:\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:", + "expected_output": "

test :smile: again :poop:
\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:

", + "text_content": "test \ud83d\ude04 again \ud83d\udca9\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:" + }, + { + "name": "translate_emoticons", + "input": ":) foo :( bar <3 with space : ) real emoji :smiley:", + "expected_output": "

:smiley: foo :slightly_frowning_face: bar :heart: with space : ) real emoji :smiley:

", + "marked_expected_output": "

:) foo :( bar <3 with space : ) real emoji :smiley:

", + "text_content": "\ud83d\ude03 foo \ud83d\ude41 bar \u2764 with space : ) real emoji \ud83d\ude03" + }, + { + "name": "translate_emoticons_whitepsace", + "input": "a:) ;)b", + "expected_output": "

a:) ;)b

", + "text_content": "a:) ;)b" + }, + { + "name": "translate_emoticons_in_code", + "input": "`:)`", + "expected_output": "

:)

", + "text_content": ":)" }, { "name": "random_emoji_1", diff --git a/zerver/lib/bugdown/__init__.py b/zerver/lib/bugdown/__init__.py index cdc414e09d..4464e702cf 100644 --- a/zerver/lib/bugdown/__init__.py +++ b/zerver/lib/bugdown/__init__.py @@ -32,6 +32,7 @@ from markdown.extensions import codehilite from zerver.lib.bugdown import fenced_code from zerver.lib.bugdown.fenced_code import FENCE_RE from zerver.lib.camo import get_camo_url +from zerver.lib.emoji import translate_emoticons, emoticon_regex from zerver.lib.mention import possible_mentions, \ possible_user_group_mentions, extract_user_group from zerver.lib.notifications import encode_stream @@ -998,6 +999,19 @@ def unicode_emoji_to_codepoint(unicode_emoji: Text) -> Text: codepoint = '0' + codepoint return codepoint +class EmoticonTranslation(markdown.inlinepatterns.Pattern): + """ Translates emoticons like `:)` into emoji like `:smile:`. """ + def handleMatch(self, match: Match[Text]) -> Optional[Element]: + # If there is `db_data` and it is false, then don't do translating. + # If there is no `db_data`, such as during tests, translate. + if db_data is not None and not db_data['translate_emoticons']: + return None + + emoticon = match.group('emoticon') + translated = translate_emoticons(emoticon) + name = translated[1:-1] + return make_emoji(name_to_codepoint[name], translated) + class UnicodeEmoji(markdown.inlinepatterns.Pattern): def handleMatch(self, match: Match[Text]) -> Optional[Element]: orig_syntax = match.group('syntax') @@ -1578,6 +1592,7 @@ class Bugdown(markdown.Extension): Tex(r'\B(?[^\n_$](\\\$|[^$\n])*)\$\$(?!\$)\B'), '>backtick') md.inlinePatterns.add('emoji', Emoji(EMOJI_REGEX), '_end') + md.inlinePatterns.add('translate_emoticons', EmoticonTranslation(emoticon_regex), '>emoji') md.inlinePatterns.add('unicodeemoji', UnicodeEmoji(unicode_emoji_regex), '_end') md.inlinePatterns.add('link', AtomicLinkPattern(markdown.inlinepatterns.LINK_RE, md), '>avatar') @@ -1956,6 +1971,7 @@ def do_convert(content: Text, 'realm_uri': message_realm.uri, 'sent_by_bot': sent_by_bot, 'stream_names': stream_name_info, + 'translate_emoticons': message.sender.translate_emoticons, } try: diff --git a/zerver/lib/emoji.py b/zerver/lib/emoji.py index abaeedcddb..f3d83bfc9a 100644 --- a/zerver/lib/emoji.py +++ b/zerver/lib/emoji.py @@ -14,6 +14,31 @@ from zerver.models import Reaction, Realm, RealmEmoji, UserProfile NAME_TO_CODEPOINT_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "name_to_codepoint.json") CODEPOINT_TO_NAME_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji", "codepoint_to_name.json") +# Emoticons and which emoji they should become. Duplicate emoji are allowed. +# Changes here should be mimicked in `static/js/emoji.js` +# and `templates/zerver/help/enable-emoticon-translations.md`. +EMOTICON_CONVERSIONS = { + ':)': ':smiley:', + '(:': ':smiley:', + ':(': ':slightly_frowning_face:', + '<3': ':heart:', + ':|': ':expressionless:', + ':/': ':confused:', +} + +possible_emoticons = EMOTICON_CONVERSIONS.keys() +possible_emoticon_regexes = map(re.escape, possible_emoticons) # type: ignore # AnyStr/str issues +emoticon_regex = '(?(' + ')|('.join(possible_emoticon_regexes) + '))(?![\S])' # type: ignore # annoying + +# Translates emoticons to their colon syntax, e.g. `:smiley:`. +def translate_emoticons(text: Text) -> Text: + translated = text + + for emoticon in EMOTICON_CONVERSIONS: + translated = re.sub(re.escape(emoticon), EMOTICON_CONVERSIONS[emoticon], translated) + + return translated + with open(NAME_TO_CODEPOINT_PATH) as fp: name_to_codepoint = ujson.load(fp) diff --git a/zerver/migrations/0142_userprofile_translate_emoticons.py b/zerver/migrations/0142_userprofile_translate_emoticons.py new file mode 100644 index 0000000000..9fe87d5442 --- /dev/null +++ b/zerver/migrations/0142_userprofile_translate_emoticons.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.6 on 2018-02-19 22:27 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('zerver', '0141_change_usergroup_description_to_textfield'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='translate_emoticons', + field=models.BooleanField(default=False), + ), + ] diff --git a/zerver/models.py b/zerver/models.py index e557c431a3..4c8e5fa434 100644 --- a/zerver/models.py +++ b/zerver/models.py @@ -588,6 +588,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin): default_language = models.CharField(default=u'en', max_length=MAX_LANGUAGE_ID_LENGTH) # type: Text high_contrast_mode = models.BooleanField(default=False) # type: bool night_mode = models.BooleanField(default=False) # type: bool + translate_emoticons = models.BooleanField(default=False) # type: bool # Hours to wait before sending another email to a user EMAIL_REMINDER_WAITPERIOD = 24 @@ -650,6 +651,7 @@ class UserProfile(AbstractBaseUser, PermissionsMixin): twenty_four_hour_time=bool, high_contrast_mode=bool, night_mode=bool, + translate_emoticons=bool, ) notification_setting_types = dict( diff --git a/zerver/tests/test_bugdown.py b/zerver/tests/test_bugdown.py index 8c9d88ff63..fc5f4403a4 100644 --- a/zerver/tests/test_bugdown.py +++ b/zerver/tests/test_bugdown.py @@ -4,6 +4,7 @@ from django.test import TestCase, override_settings from zerver.lib import bugdown from zerver.lib.actions import ( + do_set_user_display_setting, do_remove_realm_emoji, do_set_alert_words, get_realm, @@ -617,6 +618,16 @@ class BugdownTest(ZulipTestCase): converted = bugdown_convert(msg) self.assertEqual(converted, u'

:coffee::coffee:

') + def test_no_translate_emoticons_if_off(self) -> None: + user_profile = self.example_user('othello') + do_set_user_display_setting(user_profile, 'translate_emoticons', False) + msg = Message(sender=user_profile, sending_client=get_client("test")) + + content = u':)' + expected = u'

:)

' + converted = render_markdown(msg, content) + self.assertEqual(converted, expected) + def test_same_markup(self) -> None: msg = u'\u2615' # ☕ unicode_converted = bugdown_convert(msg) diff --git a/zerver/tests/test_home.py b/zerver/tests/test_home.py index 24c9b6e50c..b7040c6fb0 100644 --- a/zerver/tests/test_home.py +++ b/zerver/tests/test_home.py @@ -156,6 +156,7 @@ class HomeTest(ZulipTestCase): "subscriptions", "test_suite", "timezone", + "translate_emoticons", "twenty_four_hour_time", "unread_msgs", "unsubscribed", diff --git a/zerver/views/user_settings.py b/zerver/views/user_settings.py index d16e3ee30d..2427a13563 100644 --- a/zerver/views/user_settings.py +++ b/zerver/views/user_settings.py @@ -129,6 +129,7 @@ def update_display_settings_backend( twenty_four_hour_time: Optional[bool]=REQ(validator=check_bool, default=None), high_contrast_mode: Optional[bool]=REQ(validator=check_bool, default=None), night_mode: Optional[bool]=REQ(validator=check_bool, default=None), + translate_emoticons: Optional[bool]=REQ(validator=check_bool, default=None), default_language: Optional[bool]=REQ(validator=check_string, default=None), left_side_userlist: Optional[bool]=REQ(validator=check_bool, default=None), emojiset: Optional[str]=REQ(validator=check_string, default=None),