mirror of
https://github.com/zulip/zulip.git
synced 2025-11-18 12:54:58 +00:00
build_emoji: Migrate to use emoji_names.py file.
This migrates Zulip to use a dramatically better set of names and aliases for our emoji set, defined in emoji_names.py (which is in turn manually generated from our hand-curated CSV file). This should significantly improve the experience of using Zulip's emoji picker and emoji typeahead for finding what one is looking for.
This commit is contained in:
Harshit Bansal
committed by
Tim Abbott
Tim Abbott
parent
7a86e83a6f
commit
f636882e04
@@ -8,7 +8,7 @@ run_test('initialize', () => {
|
||||
|
||||
var complete_emoji_catalog = _.sortBy(emoji_picker.complete_emoji_catalog, 'name');
|
||||
assert.equal(complete_emoji_catalog.length, 9);
|
||||
assert.equal(_.keys(emoji_picker.emoji_collection).length, 1065);
|
||||
assert.equal(_.keys(emoji_picker.emoji_collection).length, 1037);
|
||||
|
||||
function assert_emoji_category(ele, icon, num) {
|
||||
assert.equal(ele.icon, icon);
|
||||
@@ -24,13 +24,13 @@ run_test('initialize', () => {
|
||||
check_emojis(false);
|
||||
}
|
||||
}
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 251);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-hashtag', 228);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-thumbs-o-up', 6);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 118);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 209);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 172);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 147);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 85);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-car', 119);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-smile-o', 208);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-lightbulb-o', 170);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-leaf', 153);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cutlery', 86);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-cog', 1);
|
||||
assert_emoji_category(complete_emoji_catalog.pop(), 'fa-soccer-ball-o', 67);
|
||||
});
|
||||
|
||||
@@ -265,7 +265,7 @@ def generate_sha1sum_emoji(zulip_path):
|
||||
ZULIP_EMOJI_DIR = os.path.join(zulip_path, 'tools', 'setup', 'emoji')
|
||||
sha = hashlib.sha1()
|
||||
|
||||
filenames = ['emoji_map.json', 'build_emoji', 'emoji_setup_utils.py']
|
||||
filenames = ['emoji_map.json', 'build_emoji', 'emoji_setup_utils.py', 'emoji_names.py']
|
||||
|
||||
for filename in filenames:
|
||||
file_path = os.path.join(ZULIP_EMOJI_DIR, filename)
|
||||
|
||||
@@ -24,7 +24,7 @@ var zulip_emoji = {
|
||||
exports.EMOTICON_CONVERSIONS = {
|
||||
':)': 'smiley',
|
||||
'(:': 'smiley',
|
||||
':(': 'slightly_frowning_face',
|
||||
':(': 'slight_frown',
|
||||
'<3': 'heart',
|
||||
':|': 'expressionless',
|
||||
':/': 'confused',
|
||||
|
||||
@@ -9,8 +9,8 @@ import ujson
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from emoji_setup_utils import generate_emoji_catalog, generate_codepoint_to_name_map, \
|
||||
get_extended_codepoint_to_name, get_extended_name_to_codepoint, get_extended_names_list, \
|
||||
get_new_emoji_dicts, emoji_names_for_picker, EMOJISETS
|
||||
generate_name_to_codepoint_map, emoji_names_for_picker, EMOJISETS
|
||||
from emoji_names import EMOJI_NAME_MAPS
|
||||
|
||||
ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../')
|
||||
sys.path.append(ZULIP_PATH)
|
||||
@@ -185,27 +185,15 @@ def setup_old_emoji_farm(cache_path, emoji_map):
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
def generate_map_files(cache_path, emoji_map, emoji_data, emoji_catalog, unified_reactions_data):
|
||||
# type: (str, Dict[str, str], List[Dict[str, Any]], Dict[str, List[str]], Dict[str, str]) -> None
|
||||
def generate_map_files(cache_path, emoji_catalog):
|
||||
# type: (str, Dict[str, List[str]]) -> None
|
||||
# This function generates the various data files consumed by webapp, mobile apps, bugdown etc.
|
||||
EMOJI_CODES_PATH = os.path.join(cache_path, 'emoji_codes.js')
|
||||
names = emoji_names_for_picker(EMOJI_NAME_MAPS)
|
||||
codepoint_to_name = generate_codepoint_to_name_map(EMOJI_NAME_MAPS)
|
||||
name_to_codepoint = generate_name_to_codepoint_map(EMOJI_NAME_MAPS)
|
||||
|
||||
# put thumbs_up before thumbs_down
|
||||
names = emoji_names_for_picker(emoji_map)
|
||||
down_index = names.index('thumbs_down')
|
||||
up_index = names.index('thumbs_up')
|
||||
names[down_index], names[up_index] = ('thumbs_up', 'thumbs_down')
|
||||
|
||||
name_to_codepoint = {name: unified_reactions_data[name] for name in names}
|
||||
codepoint_to_name = generate_codepoint_to_name_map(names, unified_reactions_data)
|
||||
|
||||
# Extend to use new emojis from iamcal dataset(See `emoji_can_be_included()` in emoji_setup_utils).
|
||||
new_emoji_dicts = get_new_emoji_dicts(unified_reactions_data, emoji_data)
|
||||
names = get_extended_names_list(names, new_emoji_dicts)
|
||||
name_to_codepoint = get_extended_name_to_codepoint(name_to_codepoint, new_emoji_dicts)
|
||||
codepoint_to_name = get_extended_codepoint_to_name(codepoint_to_name, new_emoji_dicts)
|
||||
|
||||
with open(EMOJI_CODES_PATH, 'w') as emoji_codes_file:
|
||||
EMOJI_CODES_FILE_PATH = os.path.join(cache_path, 'emoji_codes.js')
|
||||
with open(EMOJI_CODES_FILE_PATH, 'w') as emoji_codes_file:
|
||||
emoji_codes_file.write(EMOJI_CODES_FILE_TEMPLATE % {
|
||||
'names': names,
|
||||
'name_to_codepoint': name_to_codepoint,
|
||||
@@ -231,12 +219,7 @@ def dump_emojis(cache_path):
|
||||
EMOJI_DATA_FILE_PATH = os.path.join(NODE_MODULES_PATH, 'emoji-datasource-google', 'emoji.json')
|
||||
with open(EMOJI_DATA_FILE_PATH) as emoji_data_file:
|
||||
emoji_data = ujson.load(emoji_data_file)
|
||||
emoji_catalog = generate_emoji_catalog(emoji_data)
|
||||
|
||||
UNIFIED_REACTIONS_PATH = os.path.join(ZULIP_PATH, 'zerver',
|
||||
'management', 'data', 'unified_reactions.json')
|
||||
with open(UNIFIED_REACTIONS_PATH) as unified_reactions_file:
|
||||
unified_reactions_data = ujson.load(unified_reactions_file)
|
||||
emoji_catalog = generate_emoji_catalog(emoji_data, EMOJI_NAME_MAPS)
|
||||
|
||||
# Setup emoji farms.
|
||||
run(['rm', '-rf', cache_path])
|
||||
@@ -244,7 +227,7 @@ def dump_emojis(cache_path):
|
||||
setup_old_emoji_farm(cache_path, emoji_map)
|
||||
|
||||
# Generate various map files.
|
||||
generate_map_files(cache_path, emoji_map, emoji_data, emoji_catalog, unified_reactions_data)
|
||||
generate_map_files(cache_path, emoji_catalog)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,249 +1,44 @@
|
||||
# This tool contains all of the rules that we use to decide which of
|
||||
# the various emoji names in emoji-map.json we should actually use in
|
||||
# autocomplete and emoji pickers. You can't do all of them, because
|
||||
# otherwise there will be a ton of duplicates alphabetically next to
|
||||
# each other, which is confusing and looks bad (e.g. `angry` and
|
||||
# `angry_face` or `ab` and `ab_button` will always sort next to each
|
||||
# other, and you really want to just pick one). See docs/subsystems/emoji.md for
|
||||
# details on how this system works.
|
||||
# This file contains various helper functions used by `build_emoji` tool.
|
||||
# See docs/subsystems/emoji.md for details on how this system works.
|
||||
|
||||
from collections import defaultdict
|
||||
from itertools import permutations, chain
|
||||
|
||||
from typing import Any, Dict, List
|
||||
|
||||
# Emojisets that we currently support.
|
||||
EMOJISETS = ['apple', 'emojione', 'google', 'twitter']
|
||||
|
||||
# the corresponding code point will be set to exactly these names as a
|
||||
# final pass, overriding any other rules. This is useful for cases
|
||||
# where the two names are very different, users might reasonably type
|
||||
# either name and be surprised when they can't find the relevant emoji.
|
||||
whitelisted_names = [
|
||||
['date', 'calendar'], ['shirt', 'tshirt'], ['cupid', 'heart_with_arrow'],
|
||||
['tada', 'party_popper'], ['parking', 'p_button'], ['car', 'automobile'],
|
||||
['mortar_board', 'graduation_cap'], ['cd', 'optical_disc'], ['tv', 'television'],
|
||||
['sound', 'speaker_on'], ['mute', 'speaker_off'], ['antenna_bars', 'signal_strength'],
|
||||
['mag_right', 'right_pointing_magnifying_glass'], ['mag', 'left_pointing_magnifying_glass'],
|
||||
['loud_sound', 'speaker_loud'], ['rice_scene', 'moon_ceremony'],
|
||||
['fast_up_button', 'arrow_double_up'], ['fast_down_button', 'arrow_double_down'],
|
||||
['rewind', 'fast_reverse_button'], ['100', 'hundred_points'], ['muscle', 'flexed_biceps'],
|
||||
['walking', 'pedestrian'], ['email', 'envelope'], ['dart', 'direct_hit'],
|
||||
['wc', 'water_closet'], ['zap', 'high_voltage'], ['underage', 'no_one_under_eighteen'],
|
||||
['vhs', 'videocassette'], ['bangbang', 'double_exclamation_mark'],
|
||||
['gun', 'pistol'], ['hocho', 'kitchen_knife'], ['8ball', 'billiards'],
|
||||
['pray', 'folded_hands'], ['cop', 'police_officer'], ['phone', 'telephone'],
|
||||
['bee', 'honeybee'], ['lips', 'mouth'], ['boat', 'sailboat'], ['feet', 'paw_prints'],
|
||||
['uk', 'gb'], ['alien_monster', 'space_invader'], ['reverse_button', 'arrow_backward'],
|
||||
# both github and slack remove play_button, though I think this is better
|
||||
['play_button', 'arrow_forward'],
|
||||
# github/slack both get rid of shuffle_tracks_button, which seems wrong
|
||||
['shuffle_tracks_button', 'twisted_rightwards_arrows'],
|
||||
['iphone', 'mobile_phone'], # disagrees with github/slack/emojione
|
||||
# both github and slack remove {growing,beating}_heart, not sure what I think
|
||||
['heartpulse', 'growing_heart'], ['heartbeat', 'beating_heart'],
|
||||
# did remove cityscape_at_dusk from (city_sunset, cityscape_at_dusk)
|
||||
['sunset', 'city_sunrise'],
|
||||
['punch', 'oncoming_fist'], # doesn't include facepunch
|
||||
['+1', 'thumbs_up'], # doesn't include thumbsup
|
||||
['-1', 'thumbs_down'], # doesn't include thumbsdown
|
||||
# shit, hankey. slack allows poop, shit, hankey. github calls it hankey,
|
||||
# and autocompletes for poop and shit. emojione calls it poop, and
|
||||
# autocompletes for pile_of_poo and shit.
|
||||
['poop', 'pile_of_poo'],
|
||||
# github/slack remove cooking, but their emoji for this is an uncooked egg
|
||||
['egg', 'cooking'],
|
||||
# to match two_{men,women}_holding_hands
|
||||
['couple', 'man_and_woman_holding_hands'],
|
||||
# ['ocean', 'water_wave'], wave is so common that we want it to point only to :wave:
|
||||
]
|
||||
def emoji_names_for_picker(emoji_name_maps):
|
||||
# type: (Dict[str, Dict[str, Any]]) -> List[str]
|
||||
emoji_names = [] # type: List[str]
|
||||
for emoji_code, name_info in emoji_name_maps.items():
|
||||
emoji_names.append(name_info["canonical_name"])
|
||||
emoji_names.extend(name_info["aliases"])
|
||||
|
||||
# We blacklist certain names in cases where the algorithms below would
|
||||
# choose incorrectly which one to keep. For example, with `football`,
|
||||
# by default, our algorithm would pick just `football`, but we given
|
||||
# that :rugby_football: also exists, we want to keep
|
||||
# :american_football: instead. So we just remove the shorter names here.
|
||||
blacklisted_names = frozenset([
|
||||
# would be chosen by words_supersets or superstrings
|
||||
'football', # american_football
|
||||
'post_office', # european_post_office (there's also a japanese_post_office)
|
||||
'castle', # european_castle (there's also a japanese_castle)
|
||||
'chart', # chart_increasing_with_yen (should rename chart_increasing to chart)
|
||||
'loop', # double_curly_loop (should rename curly_loop to loop)
|
||||
'massage', # face_massage
|
||||
'bulb', # light_bulb
|
||||
'barber', # barber_pole
|
||||
'mens', # mens_room
|
||||
'womens', # womens_room
|
||||
'knife', # kitchen_knife (hocho also maps here)
|
||||
'notes', # musical_notes
|
||||
'beetle', # lady_beetle
|
||||
'ab', # ab_button (due to keeping a_button, due to the one_lettered() rule)
|
||||
'headphone', # headphones
|
||||
'mega', # megaphone
|
||||
'ski', # skis
|
||||
'high_heel', # high_heeled_shoe (so that it shows up when searching for shoe)
|
||||
# less confident about the following
|
||||
'dolls', # japanese_dolls
|
||||
'moon', # waxing_gibbous_moon (should rename crescent_moon to moon)
|
||||
'clapper', # clapper_board
|
||||
'traffic_light', # horizontal_traffic_light (there's also a vertical_traffic_light)
|
||||
'lantern',
|
||||
'red_paper_lantern', # izakaya_lantern (in the future we should make sure
|
||||
# red_paper_lantern finds this)
|
||||
|
||||
# would be chosen by longer
|
||||
'down_button', # arrow_down_small, I think to match the other arrow_*
|
||||
# names. Matching what github and slack do.
|
||||
'running_shoe', # athletic_shoe, both github and slack agree here.
|
||||
'running', # runner. slack has both, github has running_man and running_woman, but not runner
|
||||
'o2', # o_button
|
||||
'star2', # glowing_star
|
||||
'bright', # high_brightness, to match low_brightness, what github/slack do
|
||||
'dim_button', # low_brightness, copying github/slack
|
||||
'stars', # shooting_star. disagrees with github, slack, and emojione, but this seems better
|
||||
'nail_care', # nail_polish. Also disagrees github/slack/emojione, is nail_polish mostly an
|
||||
# american thing?
|
||||
'busstop', # bus_stop
|
||||
'tophat', # top_hat
|
||||
'old_woman', # older_woman, following github/slack/emojione on these
|
||||
'old_man', # older_man
|
||||
'blue_car', # recreational_vehicle
|
||||
'litter_in_bin_sign', # put_litter_in_its_place
|
||||
'moai', # moyai based on github/slack
|
||||
'fuelpump', # fuel_pump
|
||||
|
||||
# names not otherwise excluded by our heuristics
|
||||
'left_arrow', # arrow_left, to match other arrow_* shortnames
|
||||
'right_arrow', # arrow_right
|
||||
'up_arrow', # arrow_up
|
||||
'down_arrow', # arrow_down
|
||||
'chequered_flag', # checkered_flag
|
||||
'e_mail', # e-mail
|
||||
'non_potable_water', # non-potable_water
|
||||
'flipper', # dolphin
|
||||
])
|
||||
|
||||
## functions that take in a list of names at a codepoint and return a subset to exclude
|
||||
|
||||
def blacklisted(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
return [name for name in names if name in blacklisted_names]
|
||||
|
||||
# 1 letter names don't currently show up in our autocomplete. Maybe should
|
||||
# change our autocomplete so that a whitelist of letters do, like j (for joy), x, etc
|
||||
# github uses a, ab, etc. instead of a_button, slack doesn't have any of the [letter]_buttons
|
||||
def one_lettered(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
if len(names) == 1:
|
||||
return []
|
||||
return [name for name in names if len(name) == 1]
|
||||
|
||||
# If it is an ideograph (or katakana, but we'll probably deal with that
|
||||
# differently after 1.5), remove any names that don't have
|
||||
# ideograph/katakana in them
|
||||
def ideographless(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
has_ideographs = ['ideograph' in name.split('_') or
|
||||
'katakana' in name.split('_') for name in names]
|
||||
if not any(has_ideographs):
|
||||
return []
|
||||
return [name for name, has_ideograph in zip(names, has_ideographs) if not has_ideograph]
|
||||
|
||||
# In the absence of a good reason not to, we prefer :angry: over
|
||||
# :angry_face:, since it's shorter and communicates the same idea.
|
||||
#
|
||||
# This rule is subsumed by the longer rule, but still useful for
|
||||
# breaking up a hand review of the whitelist/blacklist decisions,
|
||||
# since these cases are much more clear than the "longer" ones.
|
||||
def word_superset(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
bags_of_words = [frozenset(name.split('_')) for name in names]
|
||||
bad_names = set()
|
||||
for i, j in permutations(list(range(len(names))), 2):
|
||||
if bags_of_words[i] < bags_of_words[j]:
|
||||
bad_names.add(names[j])
|
||||
return list(bad_names)
|
||||
|
||||
# We prefer :dog: over :dog2: if they both point to the same unicode
|
||||
# character.
|
||||
#
|
||||
# This rule is subsumed by the longer rule, but still useful for
|
||||
# breaking up a hand review of the whitelist/blacklist decisions,
|
||||
# since these cases are much more clear than the "longer" ones.
|
||||
def superstring(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
bad_names = set()
|
||||
for name1, name2 in permutations(names, 2):
|
||||
if name2[:len(name1)] == name1:
|
||||
bad_names.add(name2)
|
||||
return list(bad_names)
|
||||
|
||||
# The shorter one is usually a better name.
|
||||
def longer(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
lengths = [len(name) for name in names]
|
||||
min_length = min(lengths)
|
||||
return [name for name, length in zip(names, lengths) if length > min_length]
|
||||
|
||||
# A lot of emoji that have a color in their name aren't actually the
|
||||
# right color, which is super confusing. A big part of the reason is
|
||||
# that "black" and "white" actually mean filled-in and not-filled-in
|
||||
# to the Unicode committee, which is a poor choice by explains why
|
||||
# something with "black" in its name might be any solid color. Users
|
||||
# want the emoji to have reasonable names, though, so we have to
|
||||
# correct the names with "black" or "white" in them.
|
||||
#
|
||||
# Ones found after a few minutes of inspection, and after all the other filters
|
||||
# have been applied. Probably others remaining.
|
||||
miscolored_names = frozenset(['eight_pointed_black_star', 'large_blue_diamond',
|
||||
'small_blue_diamond'])
|
||||
def google_color_bug(names):
|
||||
# type: (List[str]) -> List[str]
|
||||
return [name for name in names if
|
||||
name[:5] == 'black' or name[:5] == 'white' or name in miscolored_names]
|
||||
|
||||
def emoji_names_for_picker(emoji_map):
|
||||
# type: (Dict[str, str]) -> List[str]
|
||||
codepoint_to_names = defaultdict(list) # type: Dict[str, List[str]]
|
||||
for name, codepoint in emoji_map.items():
|
||||
codepoint_to_names[codepoint].append(name)
|
||||
|
||||
# blacklisted must come first, followed by {one_lettered, ideographless}
|
||||
# Each function here returns a list of names to be removed from a list of names
|
||||
for func in [blacklisted, one_lettered, ideographless, word_superset,
|
||||
superstring, longer, google_color_bug]:
|
||||
for codepoint, names in codepoint_to_names.items():
|
||||
codepoint_to_names[codepoint] = [name for name in names if name not in func(names)]
|
||||
|
||||
for names in whitelisted_names:
|
||||
codepoint = emoji_map[names[0]]
|
||||
for name in names:
|
||||
assert (emoji_map[name] == codepoint)
|
||||
codepoint_to_names[codepoint] = names
|
||||
|
||||
return sorted(list(chain.from_iterable(codepoint_to_names.values())))
|
||||
return sorted(emoji_names)
|
||||
|
||||
# Returns a dict from categories to list of codepoints. The list of
|
||||
# codepoints are sorted according to the `sort_order` as defined in
|
||||
# `emoji_data`.
|
||||
def generate_emoji_catalog(emoji_data):
|
||||
# type: (List[Dict[str, Any]]) -> Dict[str, List[str]]
|
||||
def generate_emoji_catalog(emoji_data, emoji_name_maps):
|
||||
# type: (List[Dict[str, Any]], Dict[str, Dict[str, Any]]) -> Dict[str, List[str]]
|
||||
sort_order = {} # type: Dict[str, int]
|
||||
emoji_catalog = {} # type: Dict[str, List[str]]
|
||||
for emoji in emoji_data:
|
||||
if not emoji_is_universal(emoji):
|
||||
emoji_catalog = defaultdict(list) # type: Dict[str, List[str]]
|
||||
|
||||
for emoji_dict in emoji_data:
|
||||
emoji_code = emoji_dict["unified"].lower()
|
||||
if not emoji_is_universal(emoji_dict) or emoji_code not in emoji_name_maps:
|
||||
continue
|
||||
category = emoji["category"]
|
||||
codepoint = emoji["unified"].lower()
|
||||
sort_order[codepoint] = emoji["sort_order"]
|
||||
if category in emoji_catalog:
|
||||
emoji_catalog[category].append(codepoint)
|
||||
else:
|
||||
emoji_catalog[category] = [codepoint, ]
|
||||
category = emoji_dict["category"]
|
||||
sort_order[emoji_code] = emoji_dict["sort_order"]
|
||||
emoji_catalog[category].append(emoji_code)
|
||||
|
||||
# Sort the emojis according to iamcal's sort order. This sorting determines the
|
||||
# order in which emojis will be displayed in emoji picker.
|
||||
for category in emoji_catalog:
|
||||
emoji_catalog[category].sort(key=lambda codepoint: sort_order[codepoint])
|
||||
return emoji_catalog
|
||||
emoji_catalog[category].sort(key=lambda emoji_code: sort_order[emoji_code])
|
||||
|
||||
return dict(emoji_catalog)
|
||||
|
||||
# Use only those names for which images are present in all
|
||||
# the emoji sets so that we can switch emoji sets seemlessly.
|
||||
@@ -254,63 +49,20 @@ def emoji_is_universal(emoji_dict):
|
||||
return False
|
||||
return True
|
||||
|
||||
def generate_codepoint_to_name_map(names, unified_reactions_data):
|
||||
# type: (List[str], Dict[str, str]) -> Dict[str, str]
|
||||
# TODO: Decide canonical names. For now, using the names
|
||||
# generated for emoji picker. In case of multiple names
|
||||
# for the same emoji, lexicographically greater name is
|
||||
# used, for example, `thumbs_up` is used and not `+1`.
|
||||
def generate_codepoint_to_name_map(emoji_name_maps):
|
||||
# type: (Dict[str, Dict[str, Any]]) -> Dict[str, str]
|
||||
codepoint_to_name = {} # type: Dict[str, str]
|
||||
for name in names:
|
||||
codepoint_to_name[unified_reactions_data[name]] = name
|
||||
for emoji_code, name_info in emoji_name_maps.items():
|
||||
codepoint_to_name[emoji_code] = name_info["canonical_name"]
|
||||
return codepoint_to_name
|
||||
|
||||
def emoji_can_be_included(emoji_dict, unified_reactions_codepoints):
|
||||
# type: (Dict[str, Any], List[str]) -> bool
|
||||
# This function returns True if an emoji in new(not included in old emoji dataset) and is
|
||||
# safe to be included. Currently emojis which are represented by a sequence of codepoints
|
||||
# or emojis with ZWJ are not to be included until we implement a mechanism for dealing with
|
||||
# their unicode versions.
|
||||
# `:fried_egg:` emoji is banned for now, due to a name collision with `:egg:` emoji in
|
||||
# `unified_reactions.json` dataset, until we completely switch to iamcal dataset.
|
||||
if emoji_dict["short_name"] == "fried_egg":
|
||||
return False
|
||||
codepoint = emoji_dict["unified"].lower()
|
||||
if '-' not in codepoint and emoji_dict["category"] != "Skin Tones" and \
|
||||
emoji_is_universal(emoji_dict) and codepoint not in unified_reactions_codepoints:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_new_emoji_dicts(unified_reactions_data, emoji_data):
|
||||
# type: (Dict[str, str], List[Dict[str, Any]]) -> List[Dict[str, Any]]
|
||||
unified_reactions_codepoints = [unified_reactions_data[name] for name in unified_reactions_data]
|
||||
new_emoji_dicts = []
|
||||
for emoji_dict in emoji_data:
|
||||
if emoji_can_be_included(emoji_dict, unified_reactions_codepoints):
|
||||
new_emoji_dicts.append(emoji_dict)
|
||||
return new_emoji_dicts
|
||||
|
||||
def get_extended_names_list(names, new_emoji_dicts):
|
||||
# type: (List[str], List[Dict[str, Any]]) -> List[str]
|
||||
extended_names_list = names[:]
|
||||
for emoji_dict in new_emoji_dicts:
|
||||
extended_names_list.append(emoji_dict["short_name"])
|
||||
return extended_names_list
|
||||
|
||||
def get_extended_name_to_codepoint(name_to_codepoint, new_emoji_dicts):
|
||||
# type: (Dict[str, str], List[Dict[str, Any]]) -> Dict[str, str]
|
||||
extended_name_to_codepoint = name_to_codepoint.copy()
|
||||
for emoji_dict in new_emoji_dicts:
|
||||
emoji_name = emoji_dict["short_name"]
|
||||
codepoint = emoji_dict["unified"].lower()
|
||||
extended_name_to_codepoint[emoji_name] = codepoint
|
||||
return extended_name_to_codepoint
|
||||
|
||||
def get_extended_codepoint_to_name(codepoint_to_name, new_emoji_dicts):
|
||||
# type: (Dict[str, str], List[Dict[str, Any]]) -> Dict[str, str]
|
||||
extended_codepoint_to_name = codepoint_to_name.copy()
|
||||
for emoji_dict in new_emoji_dicts:
|
||||
emoji_name = emoji_dict["short_name"]
|
||||
codepoint = emoji_dict["unified"].lower()
|
||||
extended_codepoint_to_name[codepoint] = emoji_name
|
||||
return extended_codepoint_to_name
|
||||
def generate_name_to_codepoint_map(emoji_name_maps):
|
||||
# type: (Dict[str, Dict[str, Any]]) -> Dict[str, str]
|
||||
name_to_codepoint = {}
|
||||
for emoji_code, name_info in emoji_name_maps.items():
|
||||
canonical_name = name_info["canonical_name"]
|
||||
aliases = name_info["aliases"]
|
||||
name_to_codepoint[canonical_name] = emoji_code
|
||||
for alias in aliases:
|
||||
name_to_codepoint[alias] = emoji_code
|
||||
return name_to_codepoint
|
||||
|
||||
@@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git"
|
||||
# Typically, adding a dependency only requires a minor version bump, and
|
||||
# removing a dependency requires a major version bump.
|
||||
|
||||
PROVISION_VERSION = '22.2'
|
||||
PROVISION_VERSION = '23.0'
|
||||
|
||||
@@ -20,7 +20,7 @@ CODEPOINT_TO_NAME_PATH = os.path.join(settings.STATIC_ROOT, "generated", "emoji"
|
||||
EMOTICON_CONVERSIONS = {
|
||||
':)': ':smiley:',
|
||||
'(:': ':smiley:',
|
||||
':(': ':slightly_frowning_face:',
|
||||
':(': ':slight_frown:',
|
||||
'<3': ':heart:',
|
||||
':|': ':expressionless:',
|
||||
':/': ':confused:',
|
||||
|
||||
30
zerver/tests/fixtures/markdown_test_cases.json
vendored
30
zerver/tests/fixtures/markdown_test_cases.json
vendored
@@ -352,9 +352,9 @@
|
||||
},
|
||||
{
|
||||
"name": "many_emoji",
|
||||
"input": "test :smile: again :poop:\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
|
||||
"expected_output": "<p>test <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
|
||||
"text_content": "test \ud83d\ude04 again \ud83d\udca9\n foobar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
|
||||
"input": "test :smile: again :poop:\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:",
|
||||
"expected_output": "<p>test <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span> again <span class=\"emoji emoji-1f4a9\" title=\"poop\">:poop:</span><br>\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:</p>",
|
||||
"text_content": "test \u263A again \ud83d\udca9\n:) foo:)bar x::y::z :wasted waste: :fakeemojithisshouldnotrender:"
|
||||
},
|
||||
{
|
||||
"name": "translate_emoticons_not_enabled",
|
||||
@@ -373,7 +373,7 @@
|
||||
{
|
||||
"name": "translate_emoticons",
|
||||
"input": ":) foo :( bar <3 with space : ) real emoji :smiley:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slightly frowning face\">:slightly_frowning_face:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span> foo <span class=\"emoji emoji-1f641\" title=\"slight frown\">:slight_frown:</span> bar <span class=\"emoji emoji-2764\" title=\"heart\">:heart:</span> with space : ) real emoji <span class=\"emoji emoji-1f603\" title=\"smiley\">:smiley:</span></p>",
|
||||
"text_content": "\ud83d\ude03 foo \ud83d\ude41 bar \u2764 with space : ) real emoji \ud83d\ude03",
|
||||
"translate_emoticons": true
|
||||
},
|
||||
@@ -432,14 +432,14 @@
|
||||
{
|
||||
"name": "emojis_without_space",
|
||||
"input": ":cat:hello:dog::rabbit:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f431\" title=\"cat\">:cat:</span>hello<span class=\"emoji emoji-1f436\" title=\"dog\">:dog:</span><span class=\"emoji emoji-1f430\" title=\"rabbit\">:rabbit:</span></p>",
|
||||
"text_content": "\ud83d\udc31hello\ud83d\udc36\ud83d\udc30"
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f408\" title=\"cat\">:cat:</span>hello<span class=\"emoji emoji-1f415\" title=\"dog\">:dog:</span><span class=\"emoji emoji-1f407\" title=\"rabbit\">:rabbit:</span></p>",
|
||||
"text_content": "\ud83d\udc08hello\ud83d\udc15\ud83d\udc07"
|
||||
},
|
||||
{
|
||||
"name": "emojis_newline",
|
||||
"input": ":cat:\n:dog:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f431\" title=\"cat\">:cat:</span><br>\n<span class=\"emoji emoji-1f436\" title=\"dog\">:dog:</span></p>",
|
||||
"text_content": "\ud83d\udc31\n\ud83d\udc36"
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f408\" title=\"cat\">:cat:</span><br>\n<span class=\"emoji emoji-1f415\" title=\"dog\">:dog:</span></p>",
|
||||
"text_content": "\ud83d\udc08\n\ud83d\udc15"
|
||||
},
|
||||
{
|
||||
"name": "not_emoji",
|
||||
@@ -468,23 +468,23 @@
|
||||
{
|
||||
"name": "miscellaneous_symbols_and_pictographs",
|
||||
"input": "Merry Christmas!!\ud83c\udf84",
|
||||
"expected_output":"<p>Merry Christmas!!<span class=\"emoji emoji-1f384\" title=\"christmas tree\">:christmas_tree:</span><\/p>",
|
||||
"expected_output":"<p>Merry Christmas!!<span class=\"emoji emoji-1f384\" title=\"holiday tree\">:holiday_tree:</span><\/p>",
|
||||
"text_content": "Merry Christmas!!\ud83c\udf84"
|
||||
},
|
||||
{
|
||||
"name": "miscellaneous_and_dingbats_emoji",
|
||||
"input": "\u2693\u2797",
|
||||
"expected_output":"<p><span class=\"emoji emoji-2693\" title=\"anchor\">:anchor:</span><span class=\"emoji emoji-2797\" title=\"heavy division sign\">:heavy_division_sign:</span><\/p>"
|
||||
"expected_output":"<p><span class=\"emoji emoji-2693\" title=\"anchor\">:anchor:</span><span class=\"emoji emoji-2797\" title=\"division\">:division:</span><\/p>"
|
||||
},
|
||||
{
|
||||
"name": "supplemental_symbols_and_pictographs",
|
||||
"input": "I am a robot \ud83e\udd16.",
|
||||
"expected_output":"<p>I am a robot <span class=\"emoji emoji-1f916\" title=\"robot face\">:robot_face:</span>.<\/p>"
|
||||
"expected_output":"<p>I am a robot <span class=\"emoji emoji-1f916\" title=\"robot\">:robot:</span>.<\/p>"
|
||||
},
|
||||
{
|
||||
"name": "miscellaneous_symbols_and_arrows",
|
||||
"input": "Black upward arrow \u2b06",
|
||||
"expected_output":"<p>Black upward arrow <span class=\"emoji emoji-2b06\" title=\"arrow up\">:arrow_up:</span><\/p>"
|
||||
"expected_output":"<p>Black upward arrow <span class=\"emoji emoji-2b06\" title=\"up\">:up:</span><\/p>"
|
||||
},
|
||||
{
|
||||
"name": "unicode_emoji_without_space",
|
||||
@@ -500,12 +500,12 @@
|
||||
{
|
||||
"name": "emoji_alongside_punctuation",
|
||||
"input": ":smile:, :smile:; :smile:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>, <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span>; <span class=\"emoji emoji-1f604\" title=\"smile\">:smile:</span></p>"
|
||||
"expected_output": "<p><span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span>, <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span>; <span class=\"emoji emoji-263a\" title=\"smile\">:smile:</span></p>"
|
||||
},
|
||||
{
|
||||
"name": "new_emoji_test",
|
||||
"input": ":avocado:, :kiwifruit:, :selfie:, :gear:, :comet:, :first_place_medal:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f951\" title=\"avocado\">:avocado:</span>, <span class=\"emoji emoji-1f95d\" title=\"kiwifruit\">:kiwifruit:</span>, <span class=\"emoji emoji-1f933\" title=\"selfie\">:selfie:</span>, <span class=\"emoji emoji-2699\" title=\"gear\">:gear:</span>, <span class=\"emoji emoji-2604\" title=\"comet\">:comet:</span>, <span class=\"emoji emoji-1f947\" title=\"first place medal\">:first_place_medal:</span></p>"
|
||||
"input": ":avocado:, :kiwi:, :selfie:, :gear:, :comet:, :gold:",
|
||||
"expected_output": "<p><span class=\"emoji emoji-1f951\" title=\"avocado\">:avocado:</span>, <span class=\"emoji emoji-1f95d\" title=\"kiwi\">:kiwi:</span>, <span class=\"emoji emoji-1f933\" title=\"selfie\">:selfie:</span>, <span class=\"emoji emoji-2699\" title=\"gear\">:gear:</span>, <span class=\"emoji emoji-2604\" title=\"comet\">:comet:</span>, <span class=\"emoji emoji-1f947\" title=\"gold\">:gold:</span></p>"
|
||||
},
|
||||
{
|
||||
"name": "emoji_pipeline_newline",
|
||||
|
||||
@@ -498,7 +498,7 @@ class BugdownTest(ZulipTestCase):
|
||||
media_tweet_html = ('<a href="http://t.co/xo7pAhK6n3" target="_blank" title="http://t.co/xo7pAhK6n3">'
|
||||
'http://twitter.com/NEVNBoston/status/421654515616849920/photo/1</a>')
|
||||
|
||||
emoji_in_tweet_html = """Zulip is <span class="emoji emoji-1f4af" title="hundred points">:hundred_points:</span>% open-source!"""
|
||||
emoji_in_tweet_html = """Zulip is <span class="emoji emoji-1f4af" title="100">:100:</span>% open-source!"""
|
||||
|
||||
def make_inline_twitter_preview(url: str, tweet_html: str, image_html: str='') -> str:
|
||||
## As of right now, all previews are mocked to be the exact same tweet
|
||||
|
||||
Reference in New Issue
Block a user