build_emoji: Migrate to use emoji_names.py file.

This migrates Zulip to use a dramatically better set of names and
aliases for our emoji set, defined in emoji_names.py (which is in turn
manually generated from our hand-curated CSV file).

This should significantly improve the experience of using Zulip's
emoji picker and emoji typeahead for finding what one is looking for.

frontend_tests/node_tests/emoji_picker.js

@@ -8,7 +8,7 @@ run_test('initialize', () => {
 
     var complete_emoji_catalog = _.sortBy(emoji_picker.complete_emoji_catalog, 'name');
     assert.equal(complete_emoji_catalog.length, 9);
-    assert.equal(_.keys(emoji_picker.emoji_collection).length, 1065);
+    assert.equal(_.keys(emoji_picker.emoji_collection).length, 1037);
 
     function assert_emoji_category(ele, icon, num) {
         assert.equal(ele.icon, icon);
@@ -24,13 +24,13 @@ run_test('initialize', () => {
             check_emojis(false);
         }
     }
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 251);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 228);
     assert_emoji_category(complete_emoji_catalog.pop(), 'fa-thumbs-o-up', 6);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 118);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 119);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 209);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 208);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 172);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 170);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 147);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 153);
-    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 85);
+    assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 86);
     assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cog', 1);
     assert_emoji_category(complete_emoji_catalog.pop(), 'fa-soccer-ball-o', 67);
 });

scripts/lib/zulip_tools.py

@@ -265,7 +265,7 @@ def generate_sha1sum_emoji(zulip_path):
     ZULIP_EMOJI_DIR = os.path.join(zulip_path, 'tools', 'setup', 'emoji')
     sha = hashlib.sha1()
 
-    filenames = ['emoji_map.json', 'build_emoji', 'emoji_setup_utils.py']
+    filenames = ['emoji_map.json', 'build_emoji', 'emoji_setup_utils.py', 'emoji_names.py']
 
     for filename in filenames:
         file_path = os.path.join(ZULIP_EMOJI_DIR, filename)

static/js/emoji.js

@@ -24,7 +24,7 @@ var zulip_emoji = {
 exports.EMOTICON_CONVERSIONS = {
     ':)': 'smiley',
     '(:': 'smiley',
-    ':(': 'slightly_frowning_face',
+    ':(': 'slight_frown',
     '<3': 'heart',
     ':|': 'expressionless',
     ':/': 'confused',

tools/setup/emoji/build_emoji

@@ -9,8 +9,8 @@ import ujson
 from typing import Any, Dict, List
 
 from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \
-    get_extended_codepoint_to_name, get_extended_name_to_codepoint, get_extended_names_list, \
+    generate_name_to_codepoint_map, emoji_names_for_picker, EMOJISETS
-    get_new_emoji_dicts, emoji_names_for_picker, EMOJISETS
+from emoji_names import EMOJI_NAME_MAPS
 
 ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
 sys.path.append(ZULIP_PATH)
@@ -185,27 +185,15 @@ def setup_old_emoji_farm(cache_path, emoji_map):
         except FileExistsError:
             pass
 
-def generate_map_files(cache_path, emoji_map, emoji_data, emoji_catalog, unified_reactions_data):
+def generate_map_files(cache_path, emoji_catalog):
-    # type: (str, Dict[str, str], List[Dict[str, Any]], Dict[str, List[str]], Dict[str, str]) -> None
+    # type: (str, Dict[str, List[str]]) -> None
     # This function generates the various data files consumed by webapp, mobile apps, bugdown etc.
-    EMOJI_CODES_PATH = os.path.join(cache_path, 'emoji_codes.js')
+    names = emoji_names_for_picker(EMOJI_NAME_MAPS)
+    codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
+    name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)
 
-    # put thumbs_up before thumbs_down
+    EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.js')
-    names = emoji_names_for_picker(emoji_map)
+    with open(EMOJI_CODES_FILE_PATH, 'w') as emoji_codes_file:
-    down_index = names.index('thumbs_down')
-    up_index = names.index('thumbs_up')
-    names[down_index], names[up_index] = ('thumbs_up', 'thumbs_down')
-
-    name_to_codepoint = {name: unified_reactions_data[name] for name in names}
-    codepoint_to_name = generate_codepoint_to_name_map(names, unified_reactions_data)
-
-    # Extend to use new emojis from iamcal dataset(See `emoji_can_be_included()` in emoji_setup_utils).
-    new_emoji_dicts = get_new_emoji_dicts(unified_reactions_data, emoji_data)
-    names = get_extended_names_list(names, new_emoji_dicts)
-    name_to_codepoint = get_extended_name_to_codepoint(name_to_codepoint, new_emoji_dicts)
-    codepoint_to_name = get_extended_codepoint_to_name(codepoint_to_name, new_emoji_dicts)
-
-    with open(EMOJI_CODES_PATH, 'w') as emoji_codes_file:
         emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % {
             'names': names,
             'name_to_codepoint': name_to_codepoint,
@@ -231,12 +219,7 @@ def dump_emojis(cache_path):
     EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-google', 'emoji.json')
     with open(EMOJI_DATA_FILE_PATH) as emoji_data_file:
         emoji_data = ujson.load(emoji_data_file)
-    emoji_catalog = generate_emoji_catalog(emoji_data)
+    emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS)
-
-    UNIFIED_REACTIONS_PATH = os.path.join(ZULIP_PATH, 'zerver',
-                                          'management', 'data', 'unified_reactions.json')
-    with open(UNIFIED_REACTIONS_PATH) as unified_reactions_file:
-        unified_reactions_data = ujson.load(unified_reactions_file)
 
     # Setup emoji farms.
     run(['rm', '-rf', cache_path])
@@ -244,7 +227,7 @@ def dump_emojis(cache_path):
     setup_old_emoji_farm(cache_path, emoji_map)
 
     # Generate various map files.
-    generate_map_files(cache_path, emoji_map, emoji_data, emoji_catalog, unified_reactions_data)
+    generate_map_files(cache_path, emoji_catalog)
 
 if __name__ == "__main__":
     main()

tools/setup/emoji/emoji_setup_utils.py

@@ -1,249 +1,44 @@
-# This tool contains all of the rules that we use to decide which of
+# This file contains various helper functions used by `build_emoji` tool.
-# the various emoji names in emoji-map.json we should actually use in
+# See docs/subsystems/emoji.md for details on how this system works.
-# autocomplete and emoji pickers.  You can't do all of them, because
-# otherwise there will be a ton of duplicates alphabetically next to
-# each other, which is confusing and looks bad (e.g. `angry` and
-# `angry_face` or `ab` and `ab_button` will always sort next to each
-# other, and you really want to just pick one).  See docs/subsystems/emoji.md for
-# details on how this system works.
 
 from collections import defaultdict
-from itertools import permutations, chain
 
 from typing import Any, Dict, List
 
 # Emojisets that we currently support.
 EMOJISETS = ['apple', 'emojione', 'google', 'twitter']
 
-# the corresponding code point will be set to exactly these names as a
+def emoji_names_for_picker(emoji_name_maps):
-# final pass, overriding any other rules.  This is useful for cases
+    # type: (Dict[str, Dict[str, Any]]) -> List[str]
-# where the two names are very different, users might reasonably type
+    emoji_names = []  # type: List[str]
-# either name and be surprised when they can't find the relevant emoji.
+    for emoji_code, name_info in emoji_name_maps.items():
-whitelisted_names = [
+        emoji_names.append(name_info["canonical_name"])
-    ['date', 'calendar'], ['shirt', 'tshirt'], ['cupid', 'heart_with_arrow'],
+        emoji_names.extend(name_info["aliases"])
-    ['tada', 'party_popper'], ['parking', 'p_button'], ['car', 'automobile'],
-    ['mortar_board', 'graduation_cap'], ['cd', 'optical_disc'], ['tv', 'television'],
-    ['sound', 'speaker_on'], ['mute', 'speaker_off'], ['antenna_bars', 'signal_strength'],
-    ['mag_right', 'right_pointing_magnifying_glass'], ['mag', 'left_pointing_magnifying_glass'],
-    ['loud_sound', 'speaker_loud'], ['rice_scene', 'moon_ceremony'],
-    ['fast_up_button', 'arrow_double_up'], ['fast_down_button', 'arrow_double_down'],
-    ['rewind', 'fast_reverse_button'], ['100', 'hundred_points'], ['muscle', 'flexed_biceps'],
-    ['walking', 'pedestrian'], ['email', 'envelope'], ['dart', 'direct_hit'],
-    ['wc', 'water_closet'], ['zap', 'high_voltage'], ['underage', 'no_one_under_eighteen'],
-    ['vhs', 'videocassette'], ['bangbang', 'double_exclamation_mark'],
-    ['gun', 'pistol'], ['hocho', 'kitchen_knife'], ['8ball', 'billiards'],
-    ['pray', 'folded_hands'], ['cop', 'police_officer'], ['phone', 'telephone'],
-    ['bee', 'honeybee'], ['lips', 'mouth'], ['boat', 'sailboat'], ['feet', 'paw_prints'],
-    ['uk', 'gb'], ['alien_monster', 'space_invader'], ['reverse_button', 'arrow_backward'],
-    # both github and slack remove play_button, though I think this is better
-    ['play_button', 'arrow_forward'],
-    # github/slack both get rid of shuffle_tracks_button, which seems wrong
-    ['shuffle_tracks_button', 'twisted_rightwards_arrows'],
-    ['iphone', 'mobile_phone'],  # disagrees with github/slack/emojione
-    # both github and slack remove {growing,beating}_heart, not sure what I think
-    ['heartpulse', 'growing_heart'], ['heartbeat', 'beating_heart'],
-    # did remove cityscape_at_dusk from (city_sunset, cityscape_at_dusk)
-    ['sunset', 'city_sunrise'],
-    ['punch', 'oncoming_fist'],  # doesn't include facepunch
-    ['+1', 'thumbs_up'],  # doesn't include thumbsup
-    ['-1', 'thumbs_down'],  # doesn't include thumbsdown
-    # shit, hankey. slack allows poop, shit, hankey. github calls it hankey,
-    # and autocompletes for poop and shit. emojione calls it poop, and
-    # autocompletes for pile_of_poo and shit.
-    ['poop', 'pile_of_poo'],
-    # github/slack remove cooking, but their emoji for this is an uncooked egg
-    ['egg', 'cooking'],
-    # to match two_{men,women}_holding_hands
-    ['couple', 'man_and_woman_holding_hands'],
-    # ['ocean', 'water_wave'], wave is so common that we want it to point only to :wave:
-]
 
-# We blacklist certain names in cases where the algorithms below would
+    return sorted(emoji_names)
-# choose incorrectly which one to keep.  For example, with `football`,
-# by default, our algorithm would pick just `football`, but we given
-# that :rugby_football: also exists, we want to keep
-# :american_football: instead.  So we just remove the shorter names here.
-blacklisted_names = frozenset([
-    # would be chosen by words_supersets or superstrings
-    'football',  # american_football
-    'post_office',  # european_post_office (there's also a japanese_post_office)
-    'castle',  # european_castle (there's also a japanese_castle)
-    'chart',  # chart_increasing_with_yen (should rename chart_increasing to chart)
-    'loop',  # double_curly_loop (should rename curly_loop to loop)
-    'massage',  # face_massage
-    'bulb',  # light_bulb
-    'barber',  # barber_pole
-    'mens',  # mens_room
-    'womens',  # womens_room
-    'knife',  # kitchen_knife (hocho also maps here)
-    'notes',  # musical_notes
-    'beetle',  # lady_beetle
-    'ab',  # ab_button (due to keeping a_button, due to the one_lettered() rule)
-    'headphone',  # headphones
-    'mega',  # megaphone
-    'ski',  # skis
-    'high_heel',  # high_heeled_shoe (so that it shows up when searching for shoe)
-    # less confident about the following
-    'dolls',  # japanese_dolls
-    'moon',  # waxing_gibbous_moon (should rename crescent_moon to moon)
-    'clapper',  # clapper_board
-    'traffic_light',  # horizontal_traffic_light (there's also a vertical_traffic_light)
-    'lantern',
-    'red_paper_lantern',  # izakaya_lantern (in the future we should make sure
-                          # red_paper_lantern finds this)
-
-    # would be chosen by longer
-    'down_button',  # arrow_down_small, I think to match the other arrow_*
-                    # names. Matching what github and slack do.
-    'running_shoe',  # athletic_shoe, both github and slack agree here.
-    'running',  # runner. slack has both, github has running_man and running_woman, but not runner
-    'o2',  # o_button
-    'star2',  # glowing_star
-    'bright',  # high_brightness, to match low_brightness, what github/slack do
-    'dim_button',  # low_brightness, copying github/slack
-    'stars',  # shooting_star. disagrees with github, slack, and emojione, but this seems better
-    'nail_care',  # nail_polish. Also disagrees github/slack/emojione, is nail_polish mostly an
-                 # american thing?
-    'busstop',  # bus_stop
-    'tophat',  # top_hat
-    'old_woman',  # older_woman, following github/slack/emojione on these
-    'old_man',  # older_man
-    'blue_car',  # recreational_vehicle
-    'litter_in_bin_sign',  # put_litter_in_its_place
-    'moai',  # moyai based on github/slack
-    'fuelpump',  # fuel_pump
-
-    # names not otherwise excluded by our heuristics
-    'left_arrow',  # arrow_left, to match other arrow_* shortnames
-    'right_arrow',  # arrow_right
-    'up_arrow',  # arrow_up
-    'down_arrow',  # arrow_down
-    'chequered_flag',  # checkered_flag
-    'e_mail',  # e-mail
-    'non_potable_water',  # non-potable_water
-    'flipper',  # dolphin
-])
-
-## functions that take in a list of names at a codepoint and return a subset to exclude
-
-def blacklisted(names):
-    # type: (List[str]) -> List[str]
-    return [name for name in names if name in blacklisted_names]
-
-# 1 letter names don't currently show up in our autocomplete. Maybe should
-# change our autocomplete so that a whitelist of letters do, like j (for joy), x, etc
-# github uses a, ab, etc. instead of a_button, slack doesn't have any of the [letter]_buttons
-def one_lettered(names):
-    # type: (List[str]) -> List[str]
-    if len(names) == 1:
-        return []
-    return [name for name in names if len(name) == 1]
-
-# If it is an ideograph (or katakana, but we'll probably deal with that
-# differently after 1.5), remove any names that don't have
-# ideograph/katakana in them
-def ideographless(names):
-    # type: (List[str]) -> List[str]
-    has_ideographs = ['ideograph' in name.split('_') or
-                      'katakana' in name.split('_') for name in names]
-    if not any(has_ideographs):
-        return []
-    return [name for name, has_ideograph in zip(names, has_ideographs) if not has_ideograph]
-
-# In the absence of a good reason not to, we prefer :angry: over
-# :angry_face:, since it's shorter and communicates the same idea.
-#
-# This rule is subsumed by the longer rule, but still useful for
-# breaking up a hand review of the whitelist/blacklist decisions,
-# since these cases are much more clear than the "longer" ones.
-def word_superset(names):
-    # type: (List[str]) -> List[str]
-    bags_of_words = [frozenset(name.split('_')) for name in names]
-    bad_names = set()
-    for i, j in permutations(list(range(len(names))), 2):
-        if bags_of_words[i] < bags_of_words[j]:
-            bad_names.add(names[j])
-    return list(bad_names)
-
-# We prefer :dog: over :dog2: if they both point to the same unicode
-# character.
-#
-# This rule is subsumed by the longer rule, but still useful for
-# breaking up a hand review of the whitelist/blacklist decisions,
-# since these cases are much more clear than the "longer" ones.
-def superstring(names):
-    # type: (List[str]) -> List[str]
-    bad_names = set()
-    for name1, name2 in permutations(names, 2):
-        if name2[:len(name1)] == name1:
-            bad_names.add(name2)
-    return list(bad_names)
-
-# The shorter one is usually a better name.
-def longer(names):
-    # type: (List[str]) -> List[str]
-    lengths = [len(name) for name in names]
-    min_length = min(lengths)
-    return [name for name, length in zip(names, lengths) if length > min_length]
-
-# A lot of emoji that have a color in their name aren't actually the
-# right color, which is super confusing.  A big part of the reason is
-# that "black" and "white" actually mean filled-in and not-filled-in
-# to the Unicode committee, which is a poor choice by explains why
-# something with "black" in its name might be any solid color.  Users
-# want the emoji to have reasonable names, though, so we have to
-# correct the names with "black" or "white" in them.
-#
-# Ones found after a few minutes of inspection, and after all the other filters
-# have been applied. Probably others remaining.
-miscolored_names = frozenset(['eight_pointed_black_star', 'large_blue_diamond',
-                              'small_blue_diamond'])
-def google_color_bug(names):
-    # type: (List[str]) -> List[str]
-    return [name for name in names if
-            name[:5] == 'black' or name[:5] == 'white' or name in miscolored_names]
-
-def emoji_names_for_picker(emoji_map):
-    # type: (Dict[str, str]) -> List[str]
-    codepoint_to_names = defaultdict(list)  # type: Dict[str, List[str]]
-    for name, codepoint in emoji_map.items():
-        codepoint_to_names[codepoint].append(name)
-
-    # blacklisted must come first, followed by {one_lettered, ideographless}
-    # Each function here returns a list of names to be removed from a list of names
-    for func in [blacklisted, one_lettered, ideographless, word_superset,
-                 superstring, longer, google_color_bug]:
-        for codepoint, names in codepoint_to_names.items():
-            codepoint_to_names[codepoint] = [name for name in names if name not in func(names)]
-
-    for names in whitelisted_names:
-        codepoint = emoji_map[names[0]]
-        for name in names:
-            assert (emoji_map[name] == codepoint)
-        codepoint_to_names[codepoint] = names
-
-    return sorted(list(chain.from_iterable(codepoint_to_names.values())))
 
 # Returns a dict from categories to list of codepoints. The list of
 # codepoints are sorted according to the `sort_order` as defined in
 # `emoji_data`.
-def generate_emoji_catalog(emoji_data):
+def generate_emoji_catalog(emoji_data, emoji_name_maps):
-    # type: (List[Dict[str, Any]]) -> Dict[str, List[str]]
+    # type: (List[Dict[str, Any]], Dict[str, Dict[str, Any]]) -> Dict[str, List[str]]
     sort_order = {}  # type: Dict[str, int]
-    emoji_catalog = {}  # type: Dict[str, List[str]]
+    emoji_catalog = defaultdict(list)  # type: Dict[str, List[str]]
-    for emoji in emoji_data:
+
-        if not emoji_is_universal(emoji):
+    for emoji_dict in emoji_data:
+        emoji_code = emoji_dict["unified"].lower()
+        if not emoji_is_universal(emoji_dict) or emoji_code not in emoji_name_maps:
             continue
-        category = emoji["category"]
+        category = emoji_dict["category"]
-        codepoint = emoji["unified"].lower()
+        sort_order[emoji_code] = emoji_dict["sort_order"]
-        sort_order[codepoint] = emoji["sort_order"]
+        emoji_catalog[category].append(emoji_code)
-        if category in emoji_catalog:
+
-            emoji_catalog[category].append(codepoint)
+    # Sort the emojis according to iamcal's sort order. This sorting determines the
-        else:
+    # order in which emojis will be displayed in emoji picker.
-            emoji_catalog[category] = [codepoint, ]
     for category in emoji_catalog:
-        emoji_catalog[category].sort(key=lambda codepoint: sort_order[codepoint])
+        emoji_catalog[category].sort(key=lambda emoji_code: sort_order[emoji_code])
-    return emoji_catalog
+
+    return dict(emoji_catalog)
 
 # Use only those names for which images are present in all
 # the emoji sets so that we can switch emoji sets seemlessly.
@@ -254,63 +49,20 @@ def emoji_is_universal(emoji_dict):
             return False
     return True
 
-def generate_codepoint_to_name_map(names, unified_reactions_data):
+def generate_codepoint_to_name_map(emoji_name_maps):
-    # type: (List[str], Dict[str, str]) -> Dict[str, str]
+    # type: (Dict[str, Dict[str, Any]]) -> Dict[str, str]
-    # TODO: Decide canonical names. For now, using the names
-    # generated for emoji picker. In case of multiple names
-    # for the same emoji, lexicographically greater name is
-    # used, for example, `thumbs_up` is used and not `+1`.
     codepoint_to_name = {}  # type: Dict[str, str]
-    for name in names:
+    for emoji_code, name_info in emoji_name_maps.items():
-        codepoint_to_name[unified_reactions_data[name]] = name
+        codepoint_to_name[emoji_code] = name_info["canonical_name"]
     return codepoint_to_name
 
-def emoji_can_be_included(emoji_dict, unified_reactions_codepoints):
+def generate_name_to_codepoint_map(emoji_name_maps):
-    # type: (Dict[str, Any], List[str]) -> bool
+    # type: (Dict[str, Dict[str, Any]]) -> Dict[str, str]
-    # This function returns True if an emoji in new(not included in old emoji dataset) and is
+    name_to_codepoint = {}
-    # safe to be included. Currently emojis which are represented by a sequence of codepoints
+    for emoji_code, name_info in emoji_name_maps.items():
-    # or emojis with ZWJ are not to be included until we implement a mechanism for dealing with
+        canonical_name = name_info["canonical_name"]
-    # their unicode versions.
+        aliases = name_info["aliases"]
-    # `:fried_egg:` emoji is banned for now, due to a name collision with `:egg:` emoji in
+        name_to_codepoint[canonical_name] = emoji_code
-    # `unified_reactions.json` dataset, until we completely switch to iamcal dataset.
+        for alias in aliases:
-    if emoji_dict["short_name"] == "fried_egg":
+            name_to_codepoint[alias] = emoji_code
-        return False
+    return name_to_codepoint
-    codepoint = emoji_dict["unified"].lower()
-    if '-' not in codepoint and emoji_dict["category"] != "Skin Tones" and \
-            emoji_is_universal(emoji_dict) and codepoint not in unified_reactions_codepoints:
-        return True
-    return False
-
-def get_new_emoji_dicts(unified_reactions_data, emoji_data):
-    # type: (Dict[str, str], List[Dict[str, Any]]) -> List[Dict[str, Any]]
-    unified_reactions_codepoints = [unified_reactions_data[name] for name in unified_reactions_data]
-    new_emoji_dicts = []
-    for emoji_dict in emoji_data:
-        if emoji_can_be_included(emoji_dict, unified_reactions_codepoints):
-            new_emoji_dicts.append(emoji_dict)
-    return new_emoji_dicts
-
-def get_extended_names_list(names, new_emoji_dicts):
-    # type: (List[str], List[Dict[str, Any]]) -> List[str]
-    extended_names_list = names[:]
-    for emoji_dict in new_emoji_dicts:
-        extended_names_list.append(emoji_dict["short_name"])
-    return extended_names_list
-
-def get_extended_name_to_codepoint(name_to_codepoint, new_emoji_dicts):
-    # type: (Dict[str, str], List[Dict[str, Any]]) -> Dict[str, str]
-    extended_name_to_codepoint = name_to_codepoint.copy()
-    for emoji_dict in new_emoji_dicts:
-        emoji_name = emoji_dict["short_name"]
-        codepoint = emoji_dict["unified"].lower()
-        extended_name_to_codepoint[emoji_name] = codepoint
-    return extended_name_to_codepoint
-
-def get_extended_codepoint_to_name(codepoint_to_name, new_emoji_dicts):
-    # type: (Dict[str, str], List[Dict[str, Any]]) -> Dict[str, str]
-    extended_codepoint_to_name = codepoint_to_name.copy()
-    for emoji_dict in new_emoji_dicts:
-        emoji_name = emoji_dict["short_name"]
-        codepoint = emoji_dict["unified"].lower()
-        extended_codepoint_to_name[codepoint] = emoji_name
-    return extended_codepoint_to_name

version.py

@@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git"
 # Typically, adding a dependency only requires a minor version bump, and
 # removing a dependency requires a major version bump.
 
-PROVISION_VERSION = '22.2'
+PROVISION_VERSION = '23.0'

zerver/lib/emoji.py

@@ -20,7 +20,7 @@ CODEPOINT_TO_NAME_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji"
 EMOTICON_CONVERSIONS = {
     ':)': ':smiley:',
     '(:': ':smiley:',
-    ':(': ':slightly_frowning_face:',
+    ':(': ':slight_frown:',
     '<3': ':heart:',
     ':|': ':expressionless:',
     ':/': ':confused:',

zerver/tests/fixtures/markdown_test_cases.json

@@ -352,9 +352,9 @@
     },
     {
       "name": "many_emoji",
-      "input":  "test :smile: again :poop:\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
+      "input":  "test :smile: again :poop:\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
-      "expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
+      "expected_output": "<p>test <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
-      "text_content": "test \ud83d\ude04 again \ud83d\udca9\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
+      "text_content": "test \u263A again \ud83d\udca9\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
     },
     {
       "name": "translate_emoticons_not_enabled",
@@ -373,7 +373,7 @@
     {
       "name": "translate_emoticons",
       "input":  ":) foo :( bar <3 with space : ) real emoji :smiley:",
-      "expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slightly frowning face\">:slightly_frowning_face:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
+      "expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slight frown\">:slight_frown:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
       "text_content": "\ud83d\ude03 foo \ud83d\ude41 bar \u2764 with space : ) real emoji \ud83d\ude03",
       "translate_emoticons": true
     },
@@ -432,14 +432,14 @@
     {
       "name": "emojis_without_space",
       "input": ":cat:hello:dog::rabbit:",
-      "expected_output": "<p><span class=\"emoji emoji-1f431\" title=\"cat\">:cat:</span>hello<span class=\"emoji emoji-1f436\" title=\"dog\">:dog:</span><span class=\"emoji emoji-1f430\" title=\"rabbit\">:rabbit:</span></p>",
+      "expected_output": "<p><span class=\"emoji emoji-1f408\" title=\"cat\">:cat:</span>hello<span class=\"emoji emoji-1f415\" title=\"dog\">:dog:</span><span class=\"emoji emoji-1f407\" title=\"rabbit\">:rabbit:</span></p>",
-      "text_content": "\ud83d\udc31hello\ud83d\udc36\ud83d\udc30"
+      "text_content": "\ud83d\udc08hello\ud83d\udc15\ud83d\udc07"
     },
     {
       "name": "emojis_newline",
       "input": ":cat:\n:dog:",
-      "expected_output": "<p><span class=\"emoji emoji-1f431\" title=\"cat\">:cat:</span><br>\n<span class=\"emoji emoji-1f436\" title=\"dog\">:dog:</span></p>",
+      "expected_output": "<p><span class=\"emoji emoji-1f408\" title=\"cat\">:cat:</span><br>\n<span class=\"emoji emoji-1f415\" title=\"dog\">:dog:</span></p>",
-      "text_content": "\ud83d\udc31\n\ud83d\udc36"
+      "text_content": "\ud83d\udc08\n\ud83d\udc15"
     },
     {
       "name": "not_emoji",
@@ -468,23 +468,23 @@
     {
       "name": "miscellaneous_symbols_and_pictographs",
       "input": "Merry Christmas!!\ud83c\udf84",
-      "expected_output":"<p>Merry Christmas!!<span class=\"emoji emoji-1f384\" title=\"christmas tree\">:christmas_tree:</span><\/p>",
+      "expected_output":"<p>Merry Christmas!!<span class=\"emoji emoji-1f384\" title=\"holiday tree\">:holiday_tree:</span><\/p>",
       "text_content": "Merry Christmas!!\ud83c\udf84"
     },
     {
       "name": "miscellaneous_and_dingbats_emoji",
       "input": "\u2693\u2797",
-      "expected_output":"<p><span class=\"emoji emoji-2693\" title=\"anchor\">:anchor:</span><span class=\"emoji emoji-2797\" title=\"heavy division sign\">:heavy_division_sign:</span><\/p>"
+      "expected_output":"<p><span class=\"emoji emoji-2693\" title=\"anchor\">:anchor:</span><span class=\"emoji emoji-2797\" title=\"division\">:division:</span><\/p>"
     },
     {
       "name": "supplemental_symbols_and_pictographs",
       "input": "I am a robot \ud83e\udd16.",
-      "expected_output":"<p>I am a robot <span class=\"emoji emoji-1f916\" title=\"robot face\">:robot_face:</span>.<\/p>"
+      "expected_output":"<p>I am a robot <span class=\"emoji emoji-1f916\" title=\"robot\">:robot:</span>.<\/p>"
     },
     {
       "name": "miscellaneous_symbols_and_arrows",
       "input": "Black upward arrow \u2b06",
-      "expected_output":"<p>Black upward arrow <span class=\"emoji emoji-2b06\" title=\"arrow up\">:arrow_up:</span><\/p>"
+      "expected_output":"<p>Black upward arrow <span class=\"emoji emoji-2b06\" title=\"up\">:up:</span><\/p>"
     },
     {
       "name": "unicode_emoji_without_space",
@@ -500,12 +500,12 @@
     {
       "name": "emoji_alongside_punctuation",
       "input": ":smile:, :smile:; :smile:",
-      "expected_output": "<p><span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>, <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>; <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span></p>"
+      "expected_output": "<p><span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span>, <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span>; <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span></p>"
     },
     {
       "name": "new_emoji_test",
-      "input": ":avocado:, :kiwifruit:, :selfie:, :gear:, :comet:, :first_place_medal:",
+      "input": ":avocado:, :kiwi:, :selfie:, :gear:, :comet:, :gold:",
-      "expected_output": "<p><span class=\"emoji emoji-1f951\" title=\"avocado\">:avocado:</span>, <span class=\"emoji emoji-1f95d\" title=\"kiwifruit\">:kiwifruit:</span>, <span class=\"emoji emoji-1f933\" title=\"selfie\">:selfie:</span>, <span class=\"emoji emoji-2699\" title=\"gear\">:gear:</span>, <span class=\"emoji emoji-2604\" title=\"comet\">:comet:</span>, <span class=\"emoji emoji-1f947\" title=\"first place medal\">:first_place_medal:</span></p>"
+      "expected_output": "<p><span class=\"emoji emoji-1f951\" title=\"avocado\">:avocado:</span>, <span class=\"emoji emoji-1f95d\" title=\"kiwi\">:kiwi:</span>, <span class=\"emoji emoji-1f933\" title=\"selfie\">:selfie:</span>, <span class=\"emoji emoji-2699\" title=\"gear\">:gear:</span>, <span class=\"emoji emoji-2604\" title=\"comet\">:comet:</span>, <span class=\"emoji emoji-1f947\" title=\"gold\">:gold:</span></p>"
     },
     {
       "name": "emoji_pipeline_newline",

zerver/tests/test_bugdown.py

@@ -498,7 +498,7 @@ class BugdownTest(ZulipTestCase):
         media_tweet_html = ('<a href="http://t.co/xo7pAhK6n3" target="_blank" title="http://t.co/xo7pAhK6n3">'
                             'http://twitter.com/NEVNBoston/status/421654515616849920/photo/1</a>')
 
-        emoji_in_tweet_html = """Zulip is <span class="emoji emoji-1f4af" title="hundred points">:hundred_points:</span>% open-source!"""
+        emoji_in_tweet_html = """Zulip is <span class="emoji emoji-1f4af" title="100">:100:</span>% open-source!"""
 
         def make_inline_twitter_preview(url: str, tweet_html: str, image_html: str='') -> str:
             ## As of right now, all previews are mocked to be the exact same tweet